using MathNet.Numerics.LinearAlgebra; using OpenCvSharp; using System; using System.Collections.Generic; using System.Drawing; using System.Linq; using System.Threading; using System.Threading.Tasks; using TeamAAS.Robot.Core; using TeamAAS.Robot.Enums; using TeamAAS.Robot.Interfaces; using TeamAAS.Robot.Models; using TeamAAS.Robot.Models.Robot; using TeamAAS.Vision.Calibration.Enums; using TeamAAS.Vision.Calibration.Models; namespace TeamAAS.Vision.Calibration.Services { /// /// 相机标定算法服务(自 TeamAAS-2.0 移植,保持数值行为一致)。 /// /// 与视觉库/相机/UI 解耦: /// - 机器人动作走注入的 ; /// - 拍照取点走 (由宿主用 ICamera + 视觉引擎实现); /// - 需要人工摆放标定块的提示走 ; /// - 线性代数用 MathNet(工具坐标换算复用 Robot 层 ToolCoord),九点仿射用 OpenCvSharp。 /// public class CameraCalibrationService { /// /// 触发相机拍照并进行图像处理的回调。 /// 字段含义:IsSuccess 是否成功;X/Y 特征点像素坐标;U 特征朝向;ImageWidth/ImageHeight 图像尺寸。 /// public Func> CaptureAndProcessCallback { get; set; } /// /// 提示人工"将标定块放到中心/吸嘴下"并等待确认的回调;返回 true 表示已摆好可继续。 /// 未设置时视为自动跳过(无需人工确认)。 /// public Func> PromptPlaceCalibrationBlockCallback { get; set; } /// 模板 + 平面坐标构造一个标定点(Z/U/V/W/Hand/Local/Tool 沿用模板)。 private static RPoint MakePoint(int number, Vector p, RPoint template) { return new RPoint { Number = number, X = (float)p[0], Y = (float)p[1], Z = template.Z, U = template.U, V = template.V, W = template.W, Hand = template.Hand, Local = template.Local, Tool = template.Tool }; } private async Task PromptPlaceBlockAsync() { if (PromptPlaceCalibrationBlockCallback == null) return true; return await PromptPlaceCalibrationBlockCallback(); } /// /// 固定向下相机校准工具坐标(相机固定俯视、工具不随 J4 移动)。 /// public async Task<(bool IsSuccess, Matrix RotationMatrix, double ToolX, double ToolY, double PixelScaleX, double PixelScaleY)> CalibrateFixedDownCameraTool(CalibrationInfo calibrationInfo, IRobot robot, CancellationToken cancellationToken) { double pixelScaleX = 0; double pixelScaleY = 0; Matrix RobotCameraRotationMatrix = null; // 移动机器人至待机位置。 bool isSuccess = await robot.CalibMotionAsync(calibrationInfo.HomePoint, null); if (!isSuccess) return (false, null, 0, 0, 0, 0); if (cancellationToken.IsCancellationRequested) return (false, null, 0, 0, 0, 0); if (calibrationInfo.Pick.PickPlaceModel == PickPlaceModel.Suction) { // 打开吸气。 await robot.CalibOutIOAsync(true); } else { // 张开夹爪(张开为 false,闭合为 true)。 await robot.CalibOutIOAsync(false); } if (!await PromptPlaceBlockAsync()) return (false, null, 0, 0, 0, 0); if (calibrationInfo.Pick.PickPlaceModel == PickPlaceModel.Clamp) { // 夹紧夹爪。 await robot.CalibOutIOAsync(true); } // 计算相机与机器人坐标系的旋转矩阵。 var robotCameraRotationMatrix = await AutoIdentifyRobotCameraRotationMatrix(calibrationInfo, robot, cancellationToken); if (!robotCameraRotationMatrix.IsSuccess) return (false, null, 0, 0, 0, 0); pixelScaleX = robotCameraRotationMatrix.PixelScaleX; pixelScaleY = robotCameraRotationMatrix.PixelScaleY; RobotCameraRotationMatrix = robotCameraRotationMatrix.RotationMatrix; // 相机拍照并处理图像。 var photoResult = await CaptureAndProcessCallback(); if (!photoResult.IsSuccess) return (false, null, 0, 0, 0, 0); PointF P0 = new PointF((float)(photoResult.X), (float)(photoResult.Y)); RPoint RobotU = calibrationInfo.CenterPoint.Clone(); RobotU.U += (float)calibrationInfo.Angle; // 移动机器人至中心点。 isSuccess = await robot.CalibMotionAsync(calibrationInfo.CenterPoint, null); if (!isSuccess) return (false, null, 0, 0, 0, 0); if (cancellationToken.IsCancellationRequested) return (false, null, 0, 0, 0, 0); // 打开吸气。 await robot.CalibOutIOAsync(true); // 机器人 U 轴旋转。 isSuccess = await robot.CalibMotionAsync(RobotU, null); if (!isSuccess) return (false, null, 0, 0, 0, 0); if (cancellationToken.IsCancellationRequested) return (false, null, 0, 0, 0, 0); // 关闭吸气。 await robot.CalibOutIOAsync(false); // 移动机器人至待机位置。 isSuccess = await robot.CalibMotionAsync(calibrationInfo.HomePoint, null); if (!isSuccess) return (false, null, 0, 0, 0, 0); if (cancellationToken.IsCancellationRequested) return (false, null, 0, 0, 0, 0); // 相机拍照并处理图像。 photoResult = await CaptureAndProcessCallback(); if (!photoResult.IsSuccess) return (false, null, 0, 0, 0, 0); PointF curPixel = new PointF((float)(photoResult.X), (float)(photoResult.Y)); bool IsFinish = false; for (int i = 0; i < 10; i++) { // 计算需要移动的偏移量,逼近中心点。 double offsetx = (P0.X - curPixel.X) * pixelScaleX; double offsety = (P0.Y - curPixel.Y) * pixelScaleY; var pos = RobotCameraRotationMatrix.Inverse() * Vector.Build.Dense(new double[] { offsetx, offsety }); // 移动机器人取标定块。 isSuccess = await robot.CalibMotionAsync(RobotU, null); if (!isSuccess) return (false, null, 0, 0, 0, 0); if (cancellationToken.IsCancellationRequested) return (false, null, 0, 0, 0, 0); // 打开吸气。 await robot.CalibOutIOAsync(true); RobotU.X += (float)pos[0]; RobotU.Y += (float)pos[1]; // 机器人移动。 isSuccess = await robot.CalibMotionAsync(RobotU, null); if (!isSuccess) return (false, null, 0, 0, 0, 0); if (cancellationToken.IsCancellationRequested) return (false, null, 0, 0, 0, 0); // 关闭吸气。 await robot.CalibOutIOAsync(false); // 移动机器人至待机位置。 isSuccess = await robot.CalibMotionAsync(calibrationInfo.HomePoint, null); if (!isSuccess) return (false, null, 0, 0, 0, 0); if (cancellationToken.IsCancellationRequested) return (false, null, 0, 0, 0, 0); photoResult = await CaptureAndProcessCallback(); if (!photoResult.IsSuccess) return (false, null, 0, 0, 0, 0); curPixel = new PointF((float)(photoResult.X), (float)(photoResult.Y)); offsetx = P0.X - curPixel.X; offsety = P0.Y - curPixel.Y; if (Math.Abs(offsetx) < 1 && Math.Abs(offsety) < 1) { // 移动机器人取标定块。 isSuccess = await robot.CalibMotionAsync(RobotU, null); if (!isSuccess) return (false, null, 0, 0, 0, 0); if (cancellationToken.IsCancellationRequested) return (false, null, 0, 0, 0, 0); // 打开吸气。 await robot.CalibOutIOAsync(true); // 移动机器人至待机位置。 isSuccess = await robot.CalibMotionAsync(calibrationInfo.HomePoint, null); if (!isSuccess) return (false, null, 0, 0, 0, 0); IsFinish = true; break; } if (cancellationToken.IsCancellationRequested) return (false, null, 0, 0, 0, 0); } if (IsFinish) { Vector p0 = Vector.Build.Dense(new double[] { calibrationInfo.CenterPoint.X, calibrationInfo.CenterPoint.Y }); Vector p1 = Vector.Build.Dense(new double[] { RobotU.X, RobotU.Y }); ToolCoord toolCoord = new ToolCoord(); double u1 = calibrationInfo.CenterPoint.U; double u2 = RobotU.U; if (robot.Brand == RobotBrand.Schneider) { u1 *= -1; u2 *= -1; } toolCoord.ComputeTool(p0, p1, u1, u2); if (calibrationInfo.Tool == null) calibrationInfo.Tool = new RobotTool(); calibrationInfo.Tool.X = toolCoord.X; calibrationInfo.Tool.Y = toolCoord.Y; return (true, RobotCameraRotationMatrix, calibrationInfo.Tool.X, calibrationInfo.Tool.Y, pixelScaleX, pixelScaleY); } return (false, null, 0, 0, 0, 0); } /// /// J4 移动向下相机校准工具坐标(相机/工具随 J4 运动,需考虑 J4 旋转偏转)。 /// public async Task<(bool IsSuccess, Matrix RotationMatrix, double ToolX, double ToolY, double PixelScaleX, double PixelScaleY)> CalibrateJ4MovingDownCameraTool(CalibrationInfo calibrationInfo, IRobot robot, CancellationToken cancellationToken) { double pixelScaleX = 0; double pixelScaleY = 0; Matrix RobotCameraRotationMatrix = null; // 移动机器人至中心点。 bool isSuccess = await robot.CalibMotionAsync(calibrationInfo.CenterPoint, null); if (!isSuccess) return (false, null, 0, 0, 0, 0); // 计算相机与机器人坐标系的旋转矩阵。 var robotCameraRotationMatrix = await AutoIdentifyRobotCameraRotationMatrix(calibrationInfo, robot, cancellationToken); if (!robotCameraRotationMatrix.IsSuccess) return (false, null, 0, 0, 0, 0); pixelScaleX = robotCameraRotationMatrix.PixelScaleX; pixelScaleY = robotCameraRotationMatrix.PixelScaleY; RobotCameraRotationMatrix = robotCameraRotationMatrix.RotationMatrix; // 自动移动机器人至图像中心位置。 var autoMoveResult = await AutoMoveRobotToImageCenter(robot, RobotCameraRotationMatrix, pixelScaleX, pixelScaleY, cancellationToken); if (!autoMoveResult.IsSuccess) return (false, null, 0, 0, 0, 0); calibrationInfo.CenterPoint.X = (float)autoMoveResult.X; calibrationInfo.CenterPoint.Y = (float)autoMoveResult.Y; RPoint RobotU = calibrationInfo.CenterPoint.Clone(); RobotU.U += (float)calibrationInfo.Angle; // 机器人 U 轴旋转。 isSuccess = await robot.CalibMotionAsync(RobotU, RobotU.Z); if (!isSuccess) return (false, null, 0, 0, 0, 0); if (cancellationToken.IsCancellationRequested) return (false, null, 0, 0, 0, 0); // 相机拍照并处理图像。 var photoResult = await CaptureAndProcessCallback(); if (!photoResult.IsSuccess) return (false, null, 0, 0, 0, 0); autoMoveResult = await AutoMoveRobotToImageCenter(robot, RobotCameraRotationMatrix, pixelScaleX, pixelScaleY, cancellationToken); if (!autoMoveResult.IsSuccess) return (false, null, 0, 0, 0, 0); RobotU.X = (float)autoMoveResult.X; RobotU.Y = (float)autoMoveResult.Y; Vector p0 = Vector.Build.Dense(new double[] { calibrationInfo.CenterPoint.X, calibrationInfo.CenterPoint.Y }); Vector p1 = Vector.Build.Dense(new double[] { RobotU.X, RobotU.Y }); ToolCoord toolCoord = new ToolCoord(); double u1 = calibrationInfo.CenterPoint.U; double u2 = RobotU.U; if (robot.Brand == RobotBrand.Schneider) { u1 *= -1; u2 *= -1; } toolCoord.ComputeTool(p0, p1, u1, u2); if (calibrationInfo.Tool == null) calibrationInfo.Tool = new RobotTool(); calibrationInfo.Tool.X = toolCoord.X; calibrationInfo.Tool.Y = toolCoord.Y; return (true, RobotCameraRotationMatrix, calibrationInfo.Tool.X, calibrationInfo.Tool.Y, pixelScaleX, pixelScaleY); } /// /// 固定向上相机校准工具坐标。 /// public async Task<(bool IsSuccess, Matrix RotationMatrix, double ToolX, double ToolY, double PixelScaleX, double PixelScaleY)> CalibrateFixedUpCameraTool(CalibrationInfo calibrationInfo, IRobot robot, CancellationToken cancellationToken) { double pixelScaleX = 0; double pixelScaleY = 0; Matrix RobotCameraRotationMatrix = null; // 移动机器人至中心点。 bool isSuccess = await robot.CalibMotionAsync(calibrationInfo.CenterPoint, null); if (!isSuccess) return (false, null, 0, 0, 0, 0); if (calibrationInfo.Pick.PickPlaceModel == PickPlaceModel.Suction) { // 打开吸气。 await robot.CalibOutIOAsync(true); } else { // 张开夹爪。 await robot.CalibOutIOAsync(false); } if (!await PromptPlaceBlockAsync()) return (false, null, 0, 0, 0, 0); if (calibrationInfo.Pick.PickPlaceModel == PickPlaceModel.Clamp) { // 夹紧夹爪。 await robot.CalibOutIOAsync(true); } // 计算相机与机器人坐标系的旋转矩阵。 var robotCameraRotationMatrix = await AutoIdentifyRobotCameraRotationMatrix(calibrationInfo, robot, cancellationToken); if (!robotCameraRotationMatrix.IsSuccess) return (false, null, 0, 0, 0, 0); pixelScaleX = robotCameraRotationMatrix.PixelScaleX; pixelScaleY = robotCameraRotationMatrix.PixelScaleY; RobotCameraRotationMatrix = robotCameraRotationMatrix.RotationMatrix; // 自动移动机器人至图像中心位置。 var autoMoveResult = await AutoMoveRobotToImageCenter(robot, RobotCameraRotationMatrix, pixelScaleX, pixelScaleY, cancellationToken); if (!autoMoveResult.IsSuccess) return (false, null, 0, 0, 0, 0); calibrationInfo.CenterPoint.X = (float)autoMoveResult.X; calibrationInfo.CenterPoint.Y = (float)autoMoveResult.Y; RPoint RobotU = calibrationInfo.CenterPoint.Clone(); RobotU.U += (float)calibrationInfo.Angle; // 机器人 U 轴旋转。 isSuccess = await robot.CalibMotionAsync(RobotU, RobotU.Z); if (!isSuccess) return (false, null, 0, 0, 0, 0); if (cancellationToken.IsCancellationRequested) return (false, null, 0, 0, 0, 0); // 相机拍照并处理图像。 var photoResult = await CaptureAndProcessCallback(); if (!photoResult.IsSuccess) return (false, null, 0, 0, 0, 0); autoMoveResult = await AutoMoveRobotToImageCenter(robot, RobotCameraRotationMatrix, pixelScaleX, pixelScaleY, cancellationToken); if (!autoMoveResult.IsSuccess) return (false, null, 0, 0, 0, 0); RobotU.X = (float)autoMoveResult.X; RobotU.Y = (float)autoMoveResult.Y; Vector p0 = Vector.Build.Dense(new double[] { calibrationInfo.CenterPoint.X, calibrationInfo.CenterPoint.Y }); Vector p1 = Vector.Build.Dense(new double[] { RobotU.X, RobotU.Y }); ToolCoord toolCoord = new ToolCoord(); double u1 = calibrationInfo.CenterPoint.U; double u2 = RobotU.U; if (robot.Brand == RobotBrand.Schneider) { u1 *= -1; u2 *= -1; } toolCoord.ComputeTool(p0, p1, u1, u2); if (calibrationInfo.Tool == null) calibrationInfo.Tool = new RobotTool(); calibrationInfo.Tool.X = toolCoord.X; calibrationInfo.Tool.Y = toolCoord.Y; return (true, RobotCameraRotationMatrix, calibrationInfo.Tool.X, calibrationInfo.Tool.Y, pixelScaleX, pixelScaleY); } /// /// 通过未标定相机校准工具坐标:按 分发到对应标定方法。 /// public async Task<(bool IsSuccess, Matrix RotationMatrix, double ToolX, double ToolY, double PixelScaleX, double PixelScaleY)> CalibrateUncalibratedCameraTool(CalibrationInfo calibrationInfo, IRobot robot, CancellationToken cancellationToken) { if (calibrationInfo == null) return (false, null, 0, 0, 0, 0); if (robot == null || !robot.IsConnected) return (false, null, 0, 0, 0, 0); // 设置校准参数。 await robot.CalibParameAsync(calibrationInfo.Pick, calibrationInfo.Speed, calibrationInfo.Accel, calibrationInfo.Power, calibrationInfo.WaitSuction, calibrationInfo.WaitBlow); (bool IsSuccess, Matrix RotationMatrix, double ToolX, double ToolY, double PixelScaleX, double PixelScaleY) calibresult; if (calibrationInfo.CameraMount == CameraMount.FixedDown) { calibresult = await CalibrateFixedDownCameraTool(calibrationInfo, robot, cancellationToken); } else if (calibrationInfo.CameraMount == CameraMount.FixedUp) { calibresult = await CalibrateFixedUpCameraTool(calibrationInfo, robot, cancellationToken); } else if (calibrationInfo.CameraMount == CameraMount.MobileJ4) { calibresult = await CalibrateJ4MovingDownCameraTool(calibrationInfo, robot, cancellationToken); } else { return (false, null, 0, 0, 0, 0); } if (!calibresult.IsSuccess) return (false, null, 0, 0, 0, 0); return (true, calibresult.RotationMatrix, calibresult.ToolX, calibresult.ToolY, calibresult.PixelScaleX, calibresult.PixelScaleY); } /// /// 自动识别机器人坐标系与图像坐标系的旋转矩阵(三点法)与像素尺度。 /// public async Task<(bool IsSuccess, Matrix RotationMatrix, double PixelScaleX, double PixelScaleY)> AutoIdentifyRobotCameraRotationMatrix(CalibrationInfo calibrationInfo, IRobot robot, CancellationToken cancellationToken) { // 移动距离,单位 mm。 float distance = ((float)calibrationInfo.Height) / 3; //-------------------------------------------P0----------------------------------- bool isSuccess; if (calibrationInfo.CameraMount == CameraMount.FixedDown) { isSuccess = await robot.CalibMotionAsync(calibrationInfo.CenterPoint, null); if (!isSuccess) return (false, null, 0, 0); // 关闭吸气。 await robot.CalibOutIOAsync(false); } else { isSuccess = await robot.CalibMotionAsync(calibrationInfo.CenterPoint, calibrationInfo.CenterPoint.Z); if (!isSuccess) return (false, null, 0, 0); } RPoint RobotP0 = await robot.GetRobotPosAsync(); RPoint RobotP1 = RobotP0.Clone(); RPoint RobotP2 = RobotP0.Clone(); RobotP1.X += distance; RobotP2.Y += distance; if (calibrationInfo.CameraMount == CameraMount.FixedDown) { isSuccess = await robot.CalibMotionAsync(calibrationInfo.HomePoint, null); if (!isSuccess) return (false, null, 0, 0); } var result = await CaptureAndProcessCallback(); if (!result.IsSuccess) return (false, null, 0, 0); PointF P0 = new PointF((float)result.X, (float)result.Y); //-------------------------------------------P1----------------------------------- if (calibrationInfo.CameraMount == CameraMount.FixedDown) { isSuccess = await robot.CalibMotionAsync(RobotP0, null); if (!isSuccess) return (false, null, 0, 0); if (cancellationToken.IsCancellationRequested) return (false, null, 0, 0); await robot.CalibOutIOAsync(true); isSuccess = await robot.CalibMotionAsync(RobotP1, null); if (!isSuccess) return (false, null, 0, 0); if (cancellationToken.IsCancellationRequested) return (false, null, 0, 0); await robot.CalibOutIOAsync(false); isSuccess = await robot.CalibMotionAsync(calibrationInfo.HomePoint, null); if (!isSuccess) return (false, null, 0, 0); } else { isSuccess = await robot.CalibMotionAsync(RobotP1, RobotP1.Z); if (!isSuccess) return (false, null, 0, 0); } if (cancellationToken.IsCancellationRequested) return (false, null, 0, 0); result = await CaptureAndProcessCallback(); if (!result.IsSuccess) return (false, null, 0, 0); PointF P1 = new PointF((float)result.X, (float)result.Y); //-------------------------------------------P2----------------------------------- if (calibrationInfo.CameraMount == CameraMount.FixedDown) { isSuccess = await robot.CalibMotionAsync(RobotP1, null); if (!isSuccess) return (false, null, 0, 0); if (cancellationToken.IsCancellationRequested) return (false, null, 0, 0); await robot.CalibOutIOAsync(true); isSuccess = await robot.CalibMotionAsync(RobotP2, null); if (!isSuccess) return (false, null, 0, 0); if (cancellationToken.IsCancellationRequested) return (false, null, 0, 0); await robot.CalibOutIOAsync(false); isSuccess = await robot.CalibMotionAsync(calibrationInfo.HomePoint, null); if (!isSuccess) return (false, null, 0, 0); } else { isSuccess = await robot.CalibMotionAsync(RobotP2, RobotP2.Z); if (!isSuccess) return (false, null, 0, 0); } if (cancellationToken.IsCancellationRequested) return (false, null, 0, 0); result = await CaptureAndProcessCallback(); if (!result.IsSuccess) return (false, null, 0, 0); PointF P2 = new PointF((float)result.X, (float)result.Y); //-------------------------------------------P0-------------------------------------- if (calibrationInfo.CameraMount == CameraMount.FixedDown) { isSuccess = await robot.CalibMotionAsync(RobotP2, null); if (!isSuccess) return (false, null, 0, 0); if (cancellationToken.IsCancellationRequested) return (false, null, 0, 0); await robot.CalibOutIOAsync(true); isSuccess = await robot.CalibMotionAsync(RobotP0, null); if (!isSuccess) return (false, null, 0, 0); if (cancellationToken.IsCancellationRequested) return (false, null, 0, 0); await robot.CalibOutIOAsync(false); isSuccess = await robot.CalibMotionAsync(calibrationInfo.HomePoint, null); if (!isSuccess) return (false, null, 0, 0); } else { isSuccess = await robot.CalibMotionAsync(RobotP0, RobotP0.Z); if (!isSuccess) return (false, null, 0, 0); } if (cancellationToken.IsCancellationRequested) return (false, null, 0, 0); //-----------------------------------------计算旋转矩阵------------------------------ Matrix M = CalculateImageRotationMatrix(P0, P1, P2); //-----------------------------------------计算像素和毫米比例----------------------- Vector P11 = Vector.Build.Dense(new double[] { distance, 0 }); Vector P21 = Vector.Build.Dense(new double[] { P1.X, P1.Y }) - Vector.Build.Dense(new double[] { P0.X, P0.Y }); double PixelScaleX = Math.Abs((M * P11)[0] / P21[0]); // mm/pixel double PixelScaleY = Math.Abs((M * P11)[1] / P21[1]); // mm/pixel return (true, M, PixelScaleX, PixelScaleY); } /// /// 计算图像坐标系到机器人坐标系的旋转矩阵(三点基向量法)。 /// public Matrix CalculateImageRotationMatrix(PointF P0, PointF P1, PointF P2) { Vector Point0 = Vector.Build.Dense(new double[] { (double)P0.X, (double)P0.Y }); Vector Point1 = Vector.Build.Dense(new double[] { (double)P1.X, (double)P1.Y }); Vector Point2 = Vector.Build.Dense(new double[] { (double)P2.X, (double)P2.Y }); Vector E1 = (Point1 - Point0) / (Point1 - Point0).L2Norm(); Vector E2 = (Point2 - Point0 - E1.DotProduct(Point2 - Point0) * E1) / (Point2 - Point0 - E1.DotProduct(Point2 - Point0) * E1).L2Norm(); Matrix R = Matrix.Build.DenseOfColumnVectors(E1, E2); return R; } /// /// 自动移动机器人到图像中心位置(视觉伺服逼近)。 /// public async Task<(bool IsSuccess, double X, double Y, double U)> AutoMoveRobotToImageCenter(IRobot robot, Matrix matrix, double PixelScaleX, double PixelScaleY, CancellationToken cancellationToken) { bool IsFinish = false; var result = await CaptureAndProcessCallback(); if (!result.IsSuccess) return (false, 0, 0, 0); PointF curPixel = new PointF((float)result.X, (float)result.Y); PointF P0 = new PointF((float)(result.ImageWidth / 2), (float)(result.ImageHeight / 2)); var curpos = robot.GetRobotPos(); for (int i = 0; i < 10; i++) { double offsetx = (P0.X - curPixel.X) * PixelScaleX; double offsety = (P0.Y - curPixel.Y) * PixelScaleY; var pos = matrix.Inverse() * Vector.Build.Dense(new double[] { offsetx, offsety }); curpos.X += (float)pos[0]; curpos.Y += (float)pos[1]; bool isSuccess = await robot.CalibMotionAsync(curpos, curpos.Z); if (!isSuccess) return (false, 0, 0, 0); result = await CaptureAndProcessCallback(); if (!result.IsSuccess) return (false, 0, 0, 0); curPixel = new PointF((float)result.X, (float)result.Y); offsetx = P0.X - curPixel.X; offsety = P0.Y - curPixel.Y; if (Math.Abs(offsetx) < 1 && Math.Abs(offsety) < 1) { IsFinish = true; break; } if (cancellationToken.IsCancellationRequested) return (false, 0, 0, 0); } if (IsFinish) { var finalPos = robot.GetRobotPos(); return (true, finalPos.X, finalPos.Y, finalPos.U); } return (false, 0, 0, 0); } /// /// 自动九点标定:以中心点为原点生成 3x3 九点,逐点运动+拍照,OpenCV 求仿射,计算精度指标。 /// public async Task<(bool IsSuccess, Matrix AffineTransformationMaterial, RobotPixelPoint[] NinePoint, CalibrationResult Result)> AutoNinePointCalibration(CalibrationInfo calibrationInfo, IRobot robot, CancellationToken cancellationToken) { if (calibrationInfo == null) return (false, null, null, null); if (robot == null || !robot.IsConnected) return (false, null, null, null); double IntervalX = calibrationInfo.Width / 2; double IntervalY = calibrationInfo.Height / 2; Vector[] grid = { Vector.Build.Dense(new double[] { -IntervalX, -IntervalY }), Vector.Build.Dense(new double[] { 0, -IntervalY }), Vector.Build.Dense(new double[] { IntervalX, -IntervalY }), Vector.Build.Dense(new double[] { IntervalX, 0 }), Vector.Build.Dense(new double[] { 0, 0 }), Vector.Build.Dense(new double[] { -IntervalX, 0 }), Vector.Build.Dense(new double[] { -IntervalX, IntervalY }), Vector.Build.Dense(new double[] { 0, IntervalY }), Vector.Build.Dense(new double[] { IntervalX, IntervalY }), }; // 设置校准参数。 await robot.CalibParameAsync(calibrationInfo.Pick, calibrationInfo.Speed, calibrationInfo.Accel, calibrationInfo.Power, calibrationInfo.WaitSuction, calibrationInfo.WaitBlow); bool isSuccess; var P0 = Vector.Build.Dense(new double[] { calibrationInfo.CenterPoint.X, calibrationInfo.CenterPoint.Y }); if (robot.Brand == RobotBrand.XYZ_Platform) { // 移动机器人至中心点。 isSuccess = await robot.CalibMotionAsync(calibrationInfo.CenterPoint, null); if (!isSuccess) return (false, null, null, null); if (cancellationToken.IsCancellationRequested) return (false, null, null, null); // 计算相机与机器人坐标系的旋转矩阵。 var robotCameraRotationMatrix = await AutoIdentifyRobotCameraRotationMatrix(calibrationInfo, robot, cancellationToken); if (!robotCameraRotationMatrix.IsSuccess) return (false, null, null, null); // 自动移动机器人至图像中心位置。 var autoMoveResult = await AutoMoveRobotToImageCenter(robot, robotCameraRotationMatrix.RotationMatrix, robotCameraRotationMatrix.PixelScaleX, robotCameraRotationMatrix.PixelScaleY, cancellationToken); if (!autoMoveResult.IsSuccess) return (false, null, null, null); calibrationInfo.CenterPoint.X = (float)autoMoveResult.X; calibrationInfo.CenterPoint.Y = (float)autoMoveResult.Y; if (calibrationInfo.CameraMount != CameraMount.MobileDown_XYPlatform) { calibrationInfo.MarkPoint = calibrationInfo.CenterPoint.Clone(); } calibrationInfo.RobotCameraRotationMatrix = robotCameraRotationMatrix.RotationMatrix.ToArray(); } // 机器人坐标系与图像坐标系之间的旋转矩阵。 Matrix RobotCameraRotationMatrix = Matrix.Build.DenseOfArray(calibrationInfo.RobotCameraRotationMatrix); // 将九点相对偏移旋转到机器人坐标系并叠加中心点。 List rPoints = new List(); for (int i = 0; i < grid.Length; i++) { var p = RobotCameraRotationMatrix.Inverse() * grid[i] + P0; rPoints.Add(MakePoint(i + 1, p, calibrationInfo.CenterPoint)); } List PixelPoslist = new List(); if (calibrationInfo.CalibPoints == null) { calibrationInfo.CalibPoints = new System.Collections.ObjectModel.ObservableCollection(); } else { calibrationInfo.CalibPoints.Clear(); } // 移动相机标定时,需要先记住标定块的绝对坐标(相对机器人基坐标系)P0。 if (calibrationInfo.CameraMount == CameraMount.MobileJ4) { isSuccess = await robot.CalibMotionAsync(calibrationInfo.CenterPoint, null); if (!isSuccess) return (false, null, null, null); if (cancellationToken.IsCancellationRequested) return (false, null, null, null); // 将当前吸嘴在 Tool 0 下的坐标转换为工具坐标系下的坐标,并记录。 ToolCoord toolCoord = new ToolCoord(); toolCoord.SetTool(calibrationInfo.Tool.X, calibrationInfo.Tool.Y); Vector p = Vector.Build.Dense(new double[] { calibrationInfo.CenterPoint.X, calibrationInfo.CenterPoint.Y }); var pn = toolCoord.GetToolnCoord(p, calibrationInfo.CenterPoint.U); if (calibrationInfo.MarkPoint == null) calibrationInfo.MarkPoint = new RPoint(); calibrationInfo.MarkPoint = calibrationInfo.CenterPoint.Clone(); calibrationInfo.MarkPoint.X = (float)pn[0]; calibrationInfo.MarkPoint.Y = (float)pn[1]; } for (int i = 0; i < rPoints.Count; i++) { // 移动机器人至第 i 个标定点。 if (calibrationInfo.CameraMount == CameraMount.FixedDown) { isSuccess = await robot.CalibMotionAsync(rPoints[i], null); if (!isSuccess) return (false, null, null, null); if (cancellationToken.IsCancellationRequested) return (false, null, null, null); // 关闭吸气。 await robot.CalibOutIOAsync(false); // 移动机器人至待机位置。 isSuccess = await robot.CalibMotionAsync(calibrationInfo.HomePoint, null); if (!isSuccess) return (false, null, null, null); } else { isSuccess = await robot.CalibMotionAsync(rPoints[i], rPoints[i].Z); if (!isSuccess) return (false, null, null, null); } if (cancellationToken.IsCancellationRequested) return (false, null, null, null); // 相机拍照并处理图像。 var photoResult = await CaptureAndProcessCallback(); if (!photoResult.IsSuccess) return (false, null, null, null); PointF curPixel = new PointF((float)(photoResult.X), (float)(photoResult.Y)); PixelPoslist.Add(curPixel); if (calibrationInfo.CameraMount == CameraMount.FixedDown) { // 移动机器人至第 i 个标定点。 isSuccess = await robot.CalibMotionAsync(rPoints[i], null); if (!isSuccess) return (false, null, null, null); if (cancellationToken.IsCancellationRequested) return (false, null, null, null); // 打开吸气。 await robot.CalibOutIOAsync(true); } calibrationInfo.CalibPoints.Add(new RobotPixelPoint() { Number = i + 1, Robot = new PointF(rPoints[i].X, rPoints[i].Y), Pixel = new PointF(PixelPoslist[i].X, PixelPoslist[i].Y) }); if (cancellationToken.IsCancellationRequested) return (false, null, null, null); } if (calibrationInfo.CameraMount == CameraMount.FixedDown) { isSuccess = await robot.CalibMotionAsync(calibrationInfo.HomePoint, null); if (!isSuccess) return (false, null, null, null); } if (cancellationToken.IsCancellationRequested) return (false, null, null, null); Mat matPixel = new Mat(9, 2, MatType.CV_64F); Mat matRobot = new Mat(9, 2, MatType.CV_64F); for (int i = 0; i < 9; i++) { matPixel.Set(i, 0, PixelPoslist[i].X); matPixel.Set(i, 1, PixelPoslist[i].Y); // 移动相机模式需要根据机器人九点运动的反向变换,使 Mark 点移动到相机中心。 if (calibrationInfo.CameraMount == CameraMount.MobileJ4 || calibrationInfo.CameraMount == CameraMount.MobileDown_XYPlatform) { double ox = rPoints[i].X - rPoints[0].X; double oy = rPoints[i].Y - rPoints[0].Y; matRobot.Set(i, 0, calibrationInfo.MarkPoint.X - ox); matRobot.Set(i, 1, calibrationInfo.MarkPoint.Y - oy); } else { // 将 Tool 0 下的坐标转换为 Tool n 下的坐标。 if (calibrationInfo.Tool != null) { ToolCoord toolCoord = new ToolCoord(); toolCoord.SetTool(calibrationInfo.Tool.X, calibrationInfo.Tool.Y); double u1 = rPoints[i].U; if (robot.Brand == RobotBrand.Schneider) u1 *= -1; var trans = toolCoord.GetToolnCoord(Vector.Build.Dense(new double[] { rPoints[i].X, rPoints[i].Y }), u1); matRobot.Set(i, 0, trans[0]); matRobot.Set(i, 1, trans[1]); } else { matRobot.Set(i, 0, rPoints[i].X); matRobot.Set(i, 1, rPoints[i].Y); } } } Mat mat2d = Cv2.EstimateAffine2D(matPixel, matRobot); if (mat2d == null || mat2d.Empty()) return (false, null, null, null); double a = mat2d.Get(0, 0); double b = mat2d.Get(0, 1); double tx = mat2d.Get(0, 2); double c = mat2d.Get(1, 0); double d = mat2d.Get(1, 1); double ty = mat2d.Get(1, 2); double[,] vv = { { a, b, tx }, { c, d, ty } }; Matrix matrix = Matrix.Build.DenseOfArray(vv); calibrationInfo.AffineTransformationMaterial = matrix.ToArray(); // 2x2 仿射部分的列向量分别表示图像 X/Y 轴在机器人坐标系中的方向和比例。 double pixelScaleX = Math.Sqrt(a * a + c * c); double pixelScaleY = Math.Sqrt(b * b + d * d); double rotateFromXAxis = ToDegree(Math.Atan2(c, a)); double rotateFromYAxis = ToDegree(Math.Atan2(-b, d)); double rotate = AverageAngle(rotateFromXAxis, rotateFromYAxis); double dot = a * b + c * d; double axisLengthProduct = pixelScaleX * pixelScaleY; double axisAngle = axisLengthProduct > 0 ? ToDegree(Math.Acos(Clamp(dot / axisLengthProduct, -1, 1))) : 90; double shearAngle = axisAngle - 90; double determinant = a * d - b * c; // RMS(均方根值) 标定偏差。 double sumX = 0, sumY = 0; List differenceX = new List(); List differenceY = new List(); for (int i = 0; i < 9; i++) { var pixel = Vector.Build.Dense(new double[] { PixelPoslist[i].X, PixelPoslist[i].Y, 1 }); if (calibrationInfo.CameraMount == CameraMount.MobileJ4 || calibrationInfo.CameraMount == CameraMount.MobileDown_XYPlatform) { double ox = rPoints[i].X - rPoints[0].X; double oy = rPoints[i].Y - rPoints[0].Y; differenceX.Add((calibrationInfo.MarkPoint.X - ox) - (matrix * pixel)[0]); differenceY.Add((calibrationInfo.MarkPoint.Y - oy) - (matrix * pixel)[1]); } else { if (calibrationInfo.Tool != null) { ToolCoord toolCoord = new ToolCoord(); toolCoord.SetTool(calibrationInfo.Tool.X, calibrationInfo.Tool.Y); double u1 = rPoints[i].U; if (robot.Brand == RobotBrand.Schneider) u1 *= -1; var trans = toolCoord.GetToolnCoord(Vector.Build.Dense(new double[] { rPoints[i].X, rPoints[i].Y }), u1); differenceX.Add(trans[0] - (matrix * pixel)[0]); differenceY.Add(trans[1] - (matrix * pixel)[1]); } else { differenceX.Add(rPoints[i].X - (matrix * pixel)[0]); differenceY.Add(rPoints[i].Y - (matrix * pixel)[1]); } } sumX += Math.Pow(differenceX[differenceX.Count - 1], 2); sumY += Math.Pow(differenceY[differenceY.Count - 1], 2); } double rmsX = Math.Sqrt(sumX / 9); double rmsY = Math.Sqrt(sumY / 9); var errorDistances = differenceX.Zip(differenceY, (dx, dy) => Math.Sqrt(dx * dx + dy * dy)).ToList(); CalibrationResult calibrationResult = new CalibrationResult(); calibrationResult.ScaleX = pixelScaleX; calibrationResult.ScaleY = pixelScaleY; calibrationResult.TranslationX = tx; calibrationResult.TranslationY = ty; calibrationResult.Rotate = rotate; calibrationResult.ImageXAxisAngle = rotateFromXAxis; calibrationResult.ImageYAxisAngle = rotateFromYAxis; calibrationResult.ShearAngle = shearAngle; calibrationResult.ScaleRatio = pixelScaleY == 0 ? 0 : pixelScaleX / pixelScaleY; calibrationResult.RMSX = rmsX; calibrationResult.RMSY = rmsY; calibrationResult.RMS = Math.Sqrt(errorDistances.Select(value => value * value).Average()); calibrationResult.MeanError = errorDistances.Average(); calibrationResult.MaxErrorValue = errorDistances.Max(); calibrationResult.MeanErrorX = differenceX.Average(); calibrationResult.MeanErrorY = differenceY.Average(); calibrationResult.MaxErrorValueX = differenceX.Select(Math.Abs).Max(); calibrationResult.MaxErrorValueY = differenceY.Select(Math.Abs).Max(); calibrationResult.Determinant = determinant; calibrationInfo.CalibrationResult = calibrationResult; return (true, matrix, calibrationInfo.CalibPoints.ToArray(), calibrationResult); } /// /// 执行校准验证:在 5 个验证点上复算误差,评估标定质量。 /// public async Task<(bool IsSuccess, CalibrationTestResult Result)> ExecuteCalibrationValidation(CalibrationInfo calibrationInfo, IRobot robot, CancellationToken cancellationToken) { if (calibrationInfo == null) return (false, null); if (robot == null || !robot.IsConnected) return (false, null); double IntervalX = calibrationInfo.Width * 0.90 / 2; double IntervalY = calibrationInfo.Height * 0.90 / 2; Vector[] grid = { Vector.Build.Dense(new double[] { -IntervalX, -IntervalY }), Vector.Build.Dense(new double[] { IntervalX, -IntervalY }), Vector.Build.Dense(new double[] { 0, 0 }), Vector.Build.Dense(new double[] { -IntervalX, IntervalY }), Vector.Build.Dense(new double[] { IntervalX, IntervalY }), }; var P0 = Vector.Build.Dense(new double[] { calibrationInfo.CenterPoint.X, calibrationInfo.CenterPoint.Y }); Matrix RobotCameraRotationMatrix = Matrix.Build.DenseOfArray(calibrationInfo.RobotCameraRotationMatrix); List rPoints = new List(); for (int i = 0; i < grid.Length; i++) { var p = RobotCameraRotationMatrix.Inverse() * grid[i] + P0; rPoints.Add(MakePoint(i + 1, p, calibrationInfo.CenterPoint)); } bool isSuccess; List> TestPos = new List>(); for (int i = 0; i < rPoints.Count; i++) { if (calibrationInfo.CameraMount == CameraMount.FixedDown) { isSuccess = await robot.CalibMotionAsync(rPoints[i], null); if (!isSuccess) return (false, null); if (cancellationToken.IsCancellationRequested) return (false, null); await robot.CalibOutIOAsync(false); isSuccess = await robot.CalibMotionAsync(calibrationInfo.HomePoint, null); if (!isSuccess) return (false, null); } else { isSuccess = await robot.CalibMotionAsync(rPoints[i], rPoints[i].Z); if (!isSuccess) return (false, null); } if (cancellationToken.IsCancellationRequested) return (false, null); var photoResult = await CaptureAndProcessCallback(); if (!photoResult.IsSuccess) return (false, null); PointF pixelpos = new PointF((float)(photoResult.X), (float)(photoResult.Y)); Matrix matrix = Matrix.Build.DenseOfArray(calibrationInfo.AffineTransformationMaterial); var rpos = matrix * Vector.Build.Dense(new double[] { pixelpos.X, pixelpos.Y, 1 }); TestPos.Add(rpos); if (calibrationInfo.CameraMount == CameraMount.FixedDown) { isSuccess = await robot.CalibMotionAsync(rPoints[i], null); if (!isSuccess) return (false, null); await robot.CalibOutIOAsync(true); } if (cancellationToken.IsCancellationRequested) return (false, null); } if (calibrationInfo.CameraMount == CameraMount.FixedDown) { isSuccess = await robot.CalibMotionAsync(calibrationInfo.HomePoint, null); if (!isSuccess) return (false, null); } // RMS(均方根值)标定偏差。 double sumX = 0, sumY = 0; List differenceX = new List(); List differenceY = new List(); for (int i = 0; i < 5; i++) { if (calibrationInfo.CameraMount == CameraMount.MobileJ4 || calibrationInfo.CameraMount == CameraMount.MobileDown_XYPlatform) { double Robot_X = TestPos[i][0]; double Robot_Y = TestPos[i][1]; double curpos_x = rPoints[i].X; double curpos_y = rPoints[i].Y; double curpos_u = rPoints[i].U; double angle = Math.PI * (curpos_u - calibrationInfo.MarkPoint.U) / 180; Matrix rotationMatrix = Matrix.Build.DenseOfArray(new double[,] { { Math.Cos(angle), -Math.Sin(angle) }, { Math.Sin(angle), Math.Cos(angle) } }); Vector p3 = Vector.Build.Dense(new double[] { Robot_X - calibrationInfo.CalibPoints[0].Robot.X, Robot_Y - calibrationInfo.CalibPoints[0].Robot.Y }); Vector phere = Vector.Build.Dense(new double[] { curpos_x, curpos_y }); var cc = rotationMatrix * p3 + phere; differenceX.Add(cc[0] - calibrationInfo.MarkPoint.X); differenceY.Add(cc[1] - calibrationInfo.MarkPoint.Y); } else { if (calibrationInfo.Tool != null) { ToolCoord toolCoord = new ToolCoord(); toolCoord.SetTool(calibrationInfo.Tool.X, calibrationInfo.Tool.Y); double u1 = rPoints[i].U; if (robot.Brand == RobotBrand.Schneider) u1 *= -1; var trans = toolCoord.GetToolnCoord(Vector.Build.Dense(new double[] { rPoints[i].X, rPoints[i].Y }), u1); differenceX.Add(trans[0] - TestPos[i][0]); differenceY.Add(trans[1] - TestPos[i][1]); } else { differenceX.Add(rPoints[i].X - TestPos[i][0]); differenceY.Add(rPoints[i].Y - TestPos[i][1]); } } sumX += Math.Pow(differenceX[differenceX.Count - 1], 2); sumY += Math.Pow(differenceY[differenceY.Count - 1], 2); } double rmsX = Math.Sqrt(sumX / 5); double rmsY = Math.Sqrt(sumY / 5); CalibrationTestResult calibrationTestResult = new CalibrationTestResult(); calibrationTestResult.MaxErrorValueX = differenceX.Max(); calibrationTestResult.MinErrorValueX = differenceX.Min(); calibrationTestResult.MaxErrorValueY = differenceY.Max(); calibrationTestResult.MinErrorValueY = differenceY.Min(); calibrationTestResult.RMSEX = rmsX; calibrationTestResult.RMSEY = rmsY; return (true, calibrationTestResult); } private static double ToDegree(double radians) => radians * 180 / Math.PI; private static double AverageAngle(double angle1, double angle2) => NormalizeAngle(angle1 + NormalizeAngle(angle2 - angle1) / 2); private static double NormalizeAngle(double angle) { while (angle > 180) angle -= 360; while (angle <= -180) angle += 360; return angle; } private static double Clamp(double value, double min, double max) { if (value < min) return min; return value > max ? max : value; } } }