vision/YOLOPv2.cpp
2025-06-25 13:18:25 +08:00

522 lines
20 KiB
C++
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

#include "YOLOPv2.h"
#include <QLoggingCategory>
#include "tcp_utility.hpp"
#include <iostream>
#include <vector>
#include <map>
#include <string>
#include <cstring>
#include<QJsonObject>
#include<QJsonDocument>
#include<QThread>
/*#include <winsock2.h>
#include <ws2tcpip.h>
#include <nlohmann/json.hpp> // 需要安装nlohmann/json库
#include<json/json.h>
#pragma comment(lib, "ws2_32.lib")
using json = nlohmann::json;*/
YOLOPv2::YOLOPv2(Net_config config)
{
this->confThreshold = config.confThreshold;
this->nmsThreshold = config.nmsThreshold;
//string model_path = config.modelpath;
//std::wstring widestr = std::wstring(model_path.begin(), model_path.end());
//CUDA option set
OrtCUDAProviderOptions cuda_option;
cuda_option.device_id = 0;
cuda_option.arena_extend_strategy = 0;
cuda_option.cudnn_conv_algo_search = OrtCudnnConvAlgoSearchExhaustive;
cuda_option.gpu_mem_limit = SIZE_MAX;
cuda_option.do_copy_in_default_stream = 1;
//CUDA 加速
sessionOptions.SetIntraOpNumThreads(1);//设置线程数
sessionOptions.SetGraphOptimizationLevel(GraphOptimizationLevel::ORT_ENABLE_ALL); //函数用于设置在使用 ORT 库执行模型时应用的图优化级别。ORT_ENABLE_ALL 选项启用所有可用的优化,这可以导致更快速和更高效的执行。
sessionOptions.AppendExecutionProvider_CUDA(cuda_option);
const char *modelpath = "/home/wuxianfu/Projects/Fast-YolopV2/build/onnx/yolopv2_192x320.onnx" ;
ort_session = new Session(env, modelpath, sessionOptions);
size_t numInputNodes = ort_session->GetInputCount();
size_t numOutputNodes = ort_session->GetOutputCount();
AllocatorWithDefaultOptions allocator;
for (int i = 0; i < numInputNodes; i++)
{
//input_names.push_back(ort_session->GetInputName(i, allocator));
AllocatedStringPtr input_name_Ptr = ort_session->GetInputNameAllocated(i, allocator);
input_names.push_back(input_name_Ptr.get());
qDebug() << "Input Name: " << input_name_Ptr.get();
Ort::TypeInfo input_type_info = ort_session->GetInputTypeInfo(i);
auto input_tensor_info = input_type_info.GetTensorTypeAndShapeInfo();
auto input_dims = input_tensor_info.GetShape();
input_node_dims.push_back(input_dims);
}
for (int i = 0; i < numOutputNodes; i++)
{
//output_names.push_back(ort_session->GetOutputName(i, allocator));
AllocatedStringPtr output_name_Ptr= ort_session->GetOutputNameAllocated(i, allocator);
output_names.push_back(output_name_Ptr.get());
qDebug() << "Output Name: " << output_name_Ptr.get();
Ort::TypeInfo output_type_info = ort_session->GetOutputTypeInfo(i);
auto output_tensor_info = output_type_info.GetTensorTypeAndShapeInfo();
auto output_dims = output_tensor_info.GetShape();
output_node_dims.push_back(output_dims);
}
this->inpHeight = input_node_dims[0][2];
this->inpWidth = input_node_dims[0][3];
string classesFile = "/home/wuxianfu/Projects/Fast-YolopV2/build/coco.names";
ifstream ifs(classesFile.c_str());
string line;
while (getline(ifs, line)) this->class_names.push_back(line);
this->num_class = class_names.size();
//initial tcp 开始
//tcp::TcpClient client;
// 设置消息回调
this->tcp_client.setMessageCallback([](tcp::TcpSocket& socket, const std::vector<uint8_t>& data) {
std::string message(data.begin(), data.end());
std::cout << "Received from server: " << message << std::endl;
});
// 设置错误回调
this->tcp_client.setErrorCallback([](tcp::TcpSocket& socket, tcp::ErrorCode error) {
std::cout << "Error: " << tcp::errorCodeToString(error) << std::endl;
});
// 连接服务器
auto result = this->tcp_client.connect("192.168.124.135", 6000);
if (result == tcp::ErrorCode::Success) {
std::cout << "Connected to server" << std::endl;
// 发送消息
//this->tcp_client.send("Hello, server!");
// 让客户端运行一段时间
std::cout << "Press Enter to disconnect..." << std::endl;
std::cin.get();
} else {
std::cout << "Failed to connect to server: "
<< tcp::errorCodeToString(result) << std::endl;
}
}
void YOLOPv2::normalize_(Mat img)
{
// img.convertTo(img, CV_32F);
int row = img.rows;
int col = img.cols;
this->input_image_.resize(row * col * img.channels());
for (int c = 0; c < 3; c++)
{
for (int i = 0; i < row; i++)
{
for (int j = 0; j < col; j++)
{
float pix = img.ptr<uchar>(i)[j * 3 + 2 - c];
this->input_image_[c * row * col + i * col + j] = pix / 255.0;
}
}
}
}
void YOLOPv2::nms(vector<BoxInfo>& input_boxes)
{
sort(input_boxes.begin(), input_boxes.end(), [](BoxInfo a, BoxInfo b) { return a.score > b.score; });
vector<float> vArea(input_boxes.size());
for (int i = 0; i < int(input_boxes.size()); ++i)
{
vArea[i] = (input_boxes.at(i).x2 - input_boxes.at(i).x1 + 1)
* (input_boxes.at(i).y2 - input_boxes.at(i).y1 + 1);
}
vector<bool> isSuppressed(input_boxes.size(), false);
for (int i = 0; i < int(input_boxes.size()); ++i)
{
if (isSuppressed[i]) { continue; }
for (int j = i + 1; j < int(input_boxes.size()); ++j)
{
if (isSuppressed[j]) { continue; }
float xx1 = (max)(input_boxes[i].x1, input_boxes[j].x1);
float yy1 = (max)(input_boxes[i].y1, input_boxes[j].y1);
float xx2 = (min)(input_boxes[i].x2, input_boxes[j].x2);
float yy2 = (min)(input_boxes[i].y2, input_boxes[j].y2);
float w = (max)(float(0), xx2 - xx1 + 1);
float h = (max)(float(0), yy2 - yy1 + 1);
float inter = w * h;
float ovr = inter / (vArea[i] + vArea[j] - inter);
if (ovr >= this->nmsThreshold)
{
isSuppressed[j] = true;
}
}
}
// return post_nms;
int idx_t = 0;
input_boxes.erase(remove_if(input_boxes.begin(), input_boxes.end(), [&idx_t, &isSuppressed](const BoxInfo& f) { return isSuppressed[idx_t++]; }), input_boxes.end());
}
inline float sigmoid(float x)
{
return 1.0 / (1 + exp(-x));
}
/*
void SendMessage(const char* ip, int port, map<string,double> data) {
// 初始化Winsock
WSADATA wsaData;
if (WSAStartup(MAKEWORD(2, 2), &wsaData) != 0) {
std::cerr << "WSAStartup failed: " << WSAGetLastError() << std::endl;
return;
}
// 创建套接字
SOCKET hSocket = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP);
if (hSocket == INVALID_SOCKET) {
std::cerr << "Socket creation failed: " << WSAGetLastError() << std::endl;
WSACleanup();
return;
}
// 设置服务器地址
sockaddr_in servAddr;
servAddr.sin_family = AF_INET;
servAddr.sin_port = htons(port);
if (inet_pton(AF_INET, ip, &servAddr.sin_addr) <= 0) {
std::cerr << "Invalid address/Address not supported" << std::endl;
closesocket(hSocket);
WSACleanup();
return;
}
// 连接服务器
if (connect(hSocket, (SOCKADDR*)&servAddr, sizeof(servAddr)) == SOCKET_ERROR) {
std::cerr << "Connection failed: " << WSAGetLastError() << std::endl;
closesocket(hSocket);
WSACleanup();
return;
}
// 构造JSON消息
json message;
message["receiver"] = "ANGLE";
message["sender"] = "ANGLE";
message["type"] = 5;
json params;
for (const auto& pair: data) {
params[pair.first] = pair.second;
}
params["status"] = "OK";
message["params"] = params;
std::string jsonStr = message.dump();
// 发送消息
int bytesSent = send(hSocket, jsonStr.c_str(), (int)jsonStr.size(), 0);
if (bytesSent == SOCKET_ERROR) {
std::cerr << "Send failed: " << WSAGetLastError() << std::endl;
} else {
std::cout << "Sent " << bytesSent << " bytes: " << jsonStr << std::endl;
// 接收响应(可选)
char buffer[1024];
int bytesRecv = recv(hSocket, buffer, sizeof(buffer), 0);
if (bytesRecv > 0) {
std::cout << "Received response: " << std::string(buffer, bytesRecv) << std::endl;
}
}
// 清理资源
closesocket(hSocket);
WSACleanup();
}*/
std::vector<unsigned char> YOLOPv2::mapToByteArray(const std::map<std::string, double>& param) {
std::vector<unsigned char> byteArray;
// 写入map大小
size_t mapSize = param.size();
byteArray.insert(byteArray.end(),
reinterpret_cast<const unsigned char*>(&mapSize),
reinterpret_cast<const unsigned char*>(&mapSize) + sizeof(mapSize));
// 遍历map中的每个元素
for (const auto& pair : param) {
// 写入键的长度
size_t keyLength = pair.first.size();
byteArray.insert(byteArray.end(),
reinterpret_cast<const unsigned char*>(&keyLength),
reinterpret_cast<const unsigned char*>(&keyLength) + sizeof(keyLength));
// 写入键的内容
byteArray.insert(byteArray.end(),
pair.first.begin(),
pair.first.end());
// 写入值double类型
byteArray.insert(byteArray.end(),
reinterpret_cast<const unsigned char*>(&pair.second),
reinterpret_cast<const unsigned char*>(&pair.second) + sizeof(double));
}
return byteArray;
}
Mat YOLOPv2::detect(Mat& frame)
{
Mat dstimg;
//this->inpWidth /= 10;
//this->inpHeight/=10;
resize(frame, dstimg, Size(this->inpWidth, this->inpHeight));
this->normalize_(dstimg);
array<int64_t, 4> input_shape_{ 1, 3, this->inpHeight, this->inpWidth };
auto allocator_info = MemoryInfo::CreateCpu(OrtDeviceAllocator, OrtMemTypeDefault);
Value input_tensor_ = Value::CreateTensor<float>(allocator_info, input_image_.data(), input_image_.size(), input_shape_.data(), input_shape_.size());
// 开始推理
/*qDebug() << " output_names size: " << output_names.size()<< " sec \n";
qDebug() << " input_names: " << input_names[0]<< " sec \n";
qDebug() << " output_names: " << output_names[1]<< " sec \n";
vector<Value> ort_outputs = ort_session->Run(RunOptions{nullptr}, input_names.data(), &input_tensor_, 1, output_names.data(), output_names.size());*/
const char* inputNames[] = { "input" };//这两个值是根据netron查看onnx格式得到的输入输出名称
const char* outputNames[] = { "seg" , "ll" , "pred0" , "pred1" , "pred2" , };
vector<Value> ort_outputs = ort_session->Run(RunOptions{nullptr}, inputNames, &input_tensor_, 1, outputNames, 5);
/////generate proposals
vector<BoxInfo> generate_boxes;
float ratioh = (float)frame.rows / this->inpHeight, ratiow = (float)frame.cols / this->inpWidth;
int n = 0, q = 0, i = 0, j = 0, nout = this->class_names.size() + 5, c = 0, area = 0;
for (n = 0; n < 3; n++) ///尺度
{
int num_grid_x = (int)(this->inpWidth / this->stride[n]);
int num_grid_y = (int)(this->inpHeight / this->stride[n]);
area = num_grid_x * num_grid_y;
const float* pdata = ort_outputs[n + 2].GetTensorMutableData<float>();
for (q = 0; q < 3; q++) ///anchor数
{
const float anchor_w = this->anchors[n][q * 2];
const float anchor_h = this->anchors[n][q * 2 + 1];
for (i = 0; i < num_grid_y; i++)
{
for (j = 0; j < num_grid_x; j++)
{
float box_score = sigmoid(pdata[4 * area + i * num_grid_x + j]);
if (box_score > this->confThreshold)
{
float max_class_socre = -100000, class_socre = 0;
int max_class_id = 0;
for (c = 0; c < this->class_names.size(); c++) //// get max socre
{
class_socre = pdata[(c + 5) * area + i * num_grid_x + j];
if (class_socre > max_class_socre)
{
max_class_socre = class_socre;
max_class_id = c;
}
}
max_class_socre = sigmoid(max_class_socre) * box_score;
if (max_class_socre > this->confThreshold)
{
float cx = (sigmoid(pdata[i * num_grid_x + j]) * 2.f - 0.5f + j) * this->stride[n]; ///cx
float cy = (sigmoid(pdata[area + i * num_grid_x + j]) * 2.f - 0.5f + i) * this->stride[n]; ///cy
float w = powf(sigmoid(pdata[2 * area + i * num_grid_x + j]) * 2.f, 2.f) * anchor_w; ///w
float h = powf(sigmoid(pdata[3 * area + i * num_grid_x + j]) * 2.f, 2.f) * anchor_h; ///h
float xmin = (cx - 0.5*w)*ratiow;
float ymin = (cy - 0.5*h)*ratioh;
float xmax = (cx + 0.5*w)*ratiow;
float ymax = (cy + 0.5*h)*ratioh;
generate_boxes.push_back(BoxInfo{ xmin, ymin, xmax, ymax, max_class_socre, max_class_id });
}
}
}
}
pdata += area * nout;
}
}
nms(generate_boxes);
Mat outimg = frame.clone();
for (size_t i = 0; i < generate_boxes.size(); ++i)
{
int xmin = int(generate_boxes[i].x1);
int ymin = int(generate_boxes[i].y1);
rectangle(outimg, Point(xmin, ymin), Point(int(generate_boxes[i].x2), int(generate_boxes[i].y2)), Scalar(0, 0, 255), 2);
string label = format("%.2f", generate_boxes[i].score);
label = this->class_names[generate_boxes[i].label-1] + ":" + label;
putText(outimg, label, Point(xmin, ymin - 5), FONT_HERSHEY_SIMPLEX, 0.75, Scalar(0, 255, 0), 1);
}
const float* pdrive_area = ort_outputs[0].GetTensorMutableData<float>();
const float* plane_line = ort_outputs[1].GetTensorMutableData<float>();
area = this->inpHeight*this->inpWidth;
//int min_y = -1;
//vector<Point2f> points_L, points_R;
map<double, double> deviation; //偏移采样点距离 -> 道路中心线相对于小车位置横向偏移距离(右偏为正)(单位mm
double dev_angle; //道路中心线相对小车正前方向偏移角弧度值(右偏为正)(中心线右偏即小车相对中心线左偏)
for (i = 0; i < frame.rows; i++)
{
bool flg = false;
int left = -1, right = -1;
for (j = 0; j < frame.cols; j++)
{
const int x = int(j / ratiow);
const int y = int(i / ratioh);
if (pdrive_area[y * this->inpWidth + x] < pdrive_area[area + y * this->inpWidth + x])
{
outimg.at<Vec3b>(i, j)[0] = 0;
outimg.at<Vec3b>(i, j)[1] = 255;
outimg.at<Vec3b>(i, j)[2] = 0;
}
if (plane_line[y * this->inpWidth + x] > 0.5)
{
outimg.at<Vec3b>(i, j)[0] = 255;
outimg.at<Vec3b>(i, j)[1] = 0;
outimg.at<Vec3b>(i, j)[2] = 0;
if (!flg && j >= center && right == -1) { // 记录图像右半部分最靠左的车道线的左边缘坐标
right = j;
}
flg = true;
} else {
if (flg && j - 1 < center) { //记录图像左半部分最靠右的车道线的右边缘坐标
left = j - 1;
}
flg = false;
}
}
/*if (min_y == -1 && (left != -1 || right != -1)) {
min_y = i;
}
if (left != -1){
points_L.push_back(Point2f(left, i));
}
if (right != -1){
points_R.push_back(Point2f(right, i));
}*/
//若左右参考车道线均存在,计算并标记中心点
if (left > -1 && right > -1) {
int mid = (left + right) / 2;
for (int k = -5; k <= 5; k++) {
outimg.at<Vec3b>(i, mid+k)[0] = 0;
outimg.at<Vec3b>(i, mid+k)[1] = 0;
outimg.at<Vec3b>(i, mid+k)[2] = 255;
}
//提取偏移量
for (int k = 1; k < N-1; k++) {
if (ypos[k] == i) {
deviation[dis[k]] = (mid - center) * rate[k];
cerr<<mid<<' '<<center<<' '<<rate[k]<<' '<<deviation[dis[k]]<<endl;
}
}
}
//(需要考虑的问题 1.双车道3条线 2.拐角处曲线 3.近处显示不全 4.两条线粘连)
}
//备选方案,对左右车道线分别拟合直线并计算中心线解析式 泛化 鲁棒 目前有bug
/*if (points_L.size() > 1 && points_R.size() > 1) {
Vec4f line_L, line_R;
Mat pointsMat_L(points_L.size(), 2, CV_32F), pointsMat_R(points_R.size(), 2, CV_32F),
lineRansac_L, lineRansac_R;
for (int i = 0; i < points_L.size(); i++){
pointsMat_L.at<float>(i, 0) = points_L[i].x;
pointsMat_L.at<float>(i, 1) = points_L[i].y;
}
for (int i = 0; i < points_R.size(); i++){
pointsMat_R.at<float>(i, 0) = points_R[i].x;
pointsMat_R.at<float>(i, 1) = points_R[i].y;
}
fitLine(pointsMat_L, lineRansac_L, DIST_L2, 0, 0.01, 0.01);
fitLine(pointsMat_R, lineRansac_R, DIST_L2, 0, 0.01, 0.01);
double kL, bL, kR, bR, kM, bM; // x=ky+b
kL = lineRansac_L.at<float>(0) / (lineRansac_L.at<float>(1) + 1e-12);
bL = lineRansac_L.at<float>(2) - kL * lineRansac_L.at<float>(3);
kR = lineRansac_R.at<float>(0) / (lineRansac_R.at<float>(1) + 1e-12);
bR = lineRansac_R.at<float>(2) - kR * lineRansac_R.at<float>(3);
kM = (kL + kR) / 2;
bM = (bL + bR) / 2;
for (int i = min_y; i < frame.rows; i++) {
int mid = round(kM * i + bM);
for (int k = -5; k <= 5; k++) {
outimg.at<Vec3b>(i, mid+k)[0] = 255;
outimg.at<Vec3b>(i, mid+k)[1] = 0;
outimg.at<Vec3b>(i, mid+k)[2] = 255;
}前方xx 距离处
}
for (int k = 0; k < N; k++) {
deviation[dis[k]] = (kM * ypos[k] + bM - center) * rate[k];
}
}*/
//计算偏移角
if (deviation.size() > 1) {
int siz = deviation.size();
double sumX = 0, sumY = 0, sumXY = 0, sumXX = 0;
for (auto d: deviation) {
sumX += d.first;
sumY += d.second;
sumXY += d.first * d.second;
sumXX += d.first * d.second;
cerr << d.first << ": " << d.second <<endl; //前方xx 距离处:道路中心线相对于小车位置横向偏移距离(右偏为正)(单位mm
}
double slope = (siz * sumXY - sumX * sumY) / (siz * sumXX - sumX * sumX); //道路中心线相对小车正前方向偏移角度正切值(右偏为正)
dev_angle = atan(slope)*180/3.1415926; //道路中心线相对小车正前方向偏移角弧度值(右偏为正)
cerr << slope <<endl << dev_angle <<endl;
}
//map<string, double> params;
//params["dev_1m"] = deviation[dis[0]];
//params["dev_2m"] = deviation[dis[1]];
//params["dev_3m"] = deviation[dis[2]];
//params["dev_4m"] = deviation[dis[3]];
//params["dev_5m"] = deviation[dis[4]];
//params["dev_6m"] = deviation[dis[5]];
//params["angle"] = dev_angle;
//SendMessage("127.0.0.1", 6000, params);
//std::vector<unsigned char> byteVector = mapToByteArray(params);
// 发送消息
QJsonObject json;
json["receiver"]="DRIVER";
json["sender"]="FAST";
json["type"]=22;
QJsonObject params;
params["angle"]=dev_angle;
json["params"]=params;
QJsonDocument doc(json);
QByteArray data=QJsonDocument(json).toJson();
std::string jsonstr=data.toStdString();
std::cout << jsonstr<< std::endl;
this->tcp_client.send(jsonstr);
return outimg;
}