#include "flightlib/ros_nodes/flight_pilot.hpp" namespace flightros { FlightPilot::FlightPilot(const ros::NodeHandle& nh, const ros::NodeHandle& pnh) : nh_(nh), pnh_(pnh), scene_id_(4), unity_ready_(false), unity_render_(false), receive_id_(0), frameID(0), main_loop_freq_(50.0) { // quad initialization quad_ptr_ = std::make_shared(); // load parameters std::string cfg_path = getenv("FLIGHTMARE_PATH") + std::string("/flightlib/configs/quadrotor_ros.yaml"); YAML::Node cfg_ = YAML::LoadFile(cfg_path); loadParams(cfg_); configCamera(cfg_); quad_ptr_->setSize(quad_size_); // initialization quad_state_.setZero(); if (scene_id_ == UnityScene::NATUREFOREST) { quad_state_.x(QS::POSX) = 51; // 41-61 quad_state_.x(QS::POSY) = 108; // 98-118 quad_state_.x(QS::POSZ) = 34; } quad_ptr_->reset(quad_state_); sgm_.reset(new sgm_gpu::SgmGpu(width_, height_)); // initialize subscriber and publisher left_img_pub = nh_.advertise("RGB_image", 1); stereo_pub = nh_.advertise("stereo_image", 1); depth_pub = nh_.advertise("depth_image", 1); cam_info_pub = nh_.advertise("camera_info", 1); state_est_sub_ = nh_.subscribe(odom_topic_, 1, &FlightPilot::poseCallback, this); spawn_tree_sub_ = nh_.subscribe("/spawn_tree", 1, &FlightPilot::spawnTreeCallback, this); clear_tree_sub_ = nh_.subscribe("/clear_tree", 1, &FlightPilot::clearTreeCallback, this); save_pc_sub_ = nh_.subscribe("/save_pc", 1, &FlightPilot::savePointcloudCallback, this); timestamp = ros::Time::now(); // connect unity and setup unity setUnity(unity_render_); connectUnity(); if (!unity_ready_) ROS_ERROR("Start the gazebo and unity first!"); spawnTreesAndSavePointCloud(); timer_main_loop_ = nh_.createTimer(ros::Rate(main_loop_freq_), &FlightPilot::mainLoopCallback, this); std::cout<<"[FlightRos] Ros Node is Ready!"<spawnTrees(bounding_box_, bounding_box_origin_, avg_tree_spacing_); } void FlightPilot::clearTreeCallback(const std_msgs::Empty::ConstPtr& msg) { unity_bridge_ptr_->rmTrees(); } // If the point cloud is not saved when init, it also can be saved by this callback. void FlightPilot::savePointcloudCallback(const std_msgs::Empty::ConstPtr& msg) { Vector<3> min_corner = bounding_box_origin_ - 0.5 * bounding_box_; Vector<3> max_corner = bounding_box_origin_ + 0.5 * bounding_box_; unity_bridge_ptr_->generatePointcloud(min_corner, max_corner, ply_id_, ply_path_, scene_id_, pointcloud_resolution_); } void FlightPilot::poseCallback(const nav_msgs::Odometry::ConstPtr& msg) { quad_state_.x[QS::POSX] = (Scalar)msg->pose.pose.position.x; quad_state_.x[QS::POSY] = (Scalar)msg->pose.pose.position.y; quad_state_.x[QS::POSZ] = (Scalar)msg->pose.pose.position.z; quad_state_.x[QS::ATTW] = (Scalar)msg->pose.pose.orientation.w; quad_state_.x[QS::ATTX] = (Scalar)msg->pose.pose.orientation.x; quad_state_.x[QS::ATTY] = (Scalar)msg->pose.pose.orientation.y; quad_state_.x[QS::ATTZ] = (Scalar)msg->pose.pose.orientation.z; quad_ptr_->setState(quad_state_); timestamp = msg->header.stamp; } void FlightPilot::mainLoopCallback(const ros::TimerEvent& event) { if (!unity_render_ || !unity_ready_) return; frameID++; ros::Time timestamp_ = timestamp; unity_bridge_ptr_->getRender(frameID); // 1ms FrameID frameID_rt; unity_bridge_ptr_->handleOutput(frameID_rt); // 30ms while (frameID != frameID_rt) unity_bridge_ptr_->handleOutput(frameID_rt); cv::Mat left_img, right_img, depth_img; // publish RGB image rgb_camera_left->getRGBImage(left_img); sensor_msgs::ImagePtr img_msg = cv_bridge::CvImage(std_msgs::Header(), "bgr8", left_img).toImageMsg(); img_msg->header.stamp = timestamp_; left_img_pub.publish(img_msg); // publish camera info int width = rgb_camera_left->getWidth(); int hight = rgb_camera_left->getHeight(); float fov = rgb_camera_left->getFOV(); float cx = width / 2.0; float cy = hight / 2.0; float fx = cy / tan(0.5 * fov * M_PI / 180.0); // 他这个特殊,fov是竖直方向的 float fy = fx; boost::array K = {fx, 0.0, cx, 0.0, fy, cy, 0.0, 0.0, 1.0}; sensor_msgs::CameraInfo cam_msg; cam_msg.height = hight; cam_msg.width = width; cam_msg.K = K; cam_info_pub.publish(cam_msg); // publish depth image if (use_depth) { rgb_camera_left->getDepthMap(depth_img); depth_img = depth_img * 1000.0; depth_img = cv::min(depth_img, 20.0); sensor_msgs::ImagePtr depth_msg = cv_bridge::CvImage(std_msgs::Header(), "32FC1", depth_img).toImageMsg(); depth_msg->header.stamp = timestamp_; depth_pub.publish(depth_msg); } // publish stereo image if (use_stereo) { rgb_camera_right->getRGBImage(right_img); cv::Mat stereo_(height_, width_, CV_32FC1); computeDepthImage(left_img, right_img, &stereo_); if (use_depth) { // Complete the NaN values in the depth map, as the RealSense performs better than SGM. cv::Mat mask, mask1, mask2; cv::compare(stereo_, 0, mask1, cv::CMP_EQ); // 将 A 中为 0 的位置置为 255,其余位置置为 0 cv::compare(stereo_, 20, mask2, cv::CMP_GT); // 将 A 中大于 20 的位置置为 255,其余位置置为 0 mask = mask1 | mask2; // 将两个掩码进行逻辑或操作 depth_img.copyTo(stereo_, mask); // 将 B 中 mask 为 255 的位置的值复制到 A 中 } sensor_msgs::ImagePtr stereo_msg = cv_bridge::CvImage(std_msgs::Header(), "32FC1", stereo_).toImageMsg(); stereo_msg->header.stamp = timestamp_; stereo_pub.publish(stereo_msg); } } bool FlightPilot::setUnity(const bool render) { unity_render_ = render; if (unity_render_ && unity_bridge_ptr_ == nullptr) { // create unity bridge unity_bridge_ptr_ = UnityBridge::getInstance(); unity_bridge_ptr_->addQuadrotor(quad_ptr_); ROS_INFO("[%s] Unity Bridge is created.", pnh_.getNamespace().c_str()); } return true; } bool FlightPilot::spawnTreesAndSavePointCloud() { if (!unity_ready_ || unity_bridge_ptr_ == nullptr || !spawn_tree_) return false; unity_bridge_ptr_->spawnTrees(bounding_box_, bounding_box_origin_, avg_tree_spacing_); if (!save_pointcloud_) return true; // Saving point cloud during the testing is much time-consuming, but can be used for evaluation. // The following can be commented if evaluation is not needed. Vector<3> min_corner = bounding_box_origin_ - 0.5 * bounding_box_; Vector<3> max_corner = bounding_box_origin_ + 0.5 * bounding_box_; unity_bridge_ptr_->generatePointcloud(min_corner, max_corner, ply_id_, ply_path_, scene_id_, pointcloud_resolution_); return true; } bool FlightPilot::connectUnity() { if (!unity_render_ || unity_bridge_ptr_ == nullptr) return false; unity_ready_ = unity_bridge_ptr_->connectUnity(scene_id_); return unity_ready_; } bool FlightPilot::disconnectUnity() { if (unity_render_ && unity_bridge_ptr_ != nullptr) ; unity_bridge_ptr_->disconnectUnity(); unity_ready_ = false; } bool FlightPilot::loadParams(const YAML::Node& cfg) { // ros main_loop_freq_ = cfg["main_loop_freq"].as(); odom_topic_ = cfg["odom_topic"].as(); // camera width_ = cfg["rgb_camera_left"]["width"].as(); height_ = cfg["rgb_camera_left"]["height"].as(); fov_ = cfg["rgb_camera_left"]["fov"].as(); use_depth = cfg["rgb_camera_left"]["enable_depth"].as(); use_stereo = cfg["rgb_camera_right"]["on"].as(); // scence scene_id_ = cfg["scene_id"].as(); unity_render_ = cfg["unity_render"].as(); Scalar quad_size_i = cfg["quad_size"].as(); quad_size_ = Vector<3>(quad_size_i, quad_size_i, quad_size_i); spawn_tree_ = cfg["unity"]["spawn_trees"].as(); save_pointcloud_ = cfg["unity"]["save_pointcloud"].as(); avg_tree_spacing_ = cfg["unity"]["avg_tree_spacing"].as(); for (int i = 0; i < 3; ++i) { bounding_box_(i) = cfg["unity"]["bounding_box"][i].as(); bounding_box_origin_(i) = cfg["unity"]["bounding_box_origin"][i].as(); } pointcloud_resolution_ = cfg["unity"]["pointcloud_resolution"].as(); ply_path_ = getenv("FLIGHTMARE_PATH") + cfg["ply_path"].as(); if (!std::filesystem::exists(ply_path_)) { std::filesystem::create_directories(ply_path_); std::cout << "Directory created: " << ply_path_ << std::endl; } return true; } void FlightPilot::computeDepthImage(const cv::Mat& left_frame, const cv::Mat& right_frame, cv::Mat* const depth) { cv::Mat disparity(height_, width_, CV_8UC1); sgm_->computeDisparity(left_frame, right_frame, disparity); disparity.convertTo(disparity, CV_32FC1); // compute depth from disparity float f = (width_ / 2.0) / std::tan((M_PI * (fov_ / 180.0)) / 2.0); // depth = stereo_baseline_ * f / disparity for (int r = 0; r < height_; ++r) { for (int c = 0; c < width_; ++c) { if (disparity.at(r, c) == 0.0f) { depth->at(r, c) = 0.0f; } else if (disparity.at(r, c) == 255.0f) { depth->at(r, c) = 0.0f; } else { depth->at(r, c) = static_cast(stereo_baseline_) * f / disparity.at(r, c); } } } } bool FlightPilot::configCamera(const YAML::Node& cfg) { if (!cfg["rgb_camera_left"] || !cfg["rgb_camera_right"]) { ROS_ERROR("Cannot config stereo Camera"); return false; } // create left camera -------------------------------------------- rgb_camera_left = std::make_shared(); // load camera settings std::vector t_BC_vec = cfg["rgb_camera_left"]["t_BC"].as>(); std::vector r_BC_vec = cfg["rgb_camera_left"]["r_BC"].as>(); Vector<3> t_BC(t_BC_vec.data()); Matrix<3, 3> r_BC = (AngleAxis(r_BC_vec[0] * M_PI / 180.0, Vector<3>::UnitX()) * AngleAxis(r_BC_vec[1] * M_PI / 180.0, Vector<3>::UnitY()) * AngleAxis(r_BC_vec[2] * M_PI / 180.0, Vector<3>::UnitZ())) .toRotationMatrix(); // the rotation order has been verified // Convert the horizontal FOV (usually used) to vertical FOV (flightmare). Scalar rgb_fov_deg_ = cfg["rgb_camera_left"]["fov"].as(); double hor_fov_radians = (M_PI * (rgb_fov_deg_ / 180.0)); Scalar img_rows_ = cfg["rgb_camera_left"]["height"].as(); Scalar img_cols_ = cfg["rgb_camera_left"]["width"].as(); double flightmare_fov = 2. * std::atan(std::tan(hor_fov_radians / 2) * img_rows_ / img_cols_); flightmare_fov = (flightmare_fov / M_PI) * 180.0; rgb_camera_left->setFOV(flightmare_fov); rgb_camera_left->setWidth(cfg["rgb_camera_left"]["width"].as()); rgb_camera_left->setHeight(cfg["rgb_camera_left"]["height"].as()); rgb_camera_left->setRelPose(t_BC, r_BC); rgb_camera_left->enableOpticalFlow(cfg["rgb_camera_left"]["enable_opticalflow"].as()); rgb_camera_left->enableSegmentation(cfg["rgb_camera_left"]["enable_segmentation"].as()); rgb_camera_left->enableDepth(cfg["rgb_camera_left"]["enable_depth"].as()); // add camera to the quadrotor quad_ptr_->addRGBCamera(rgb_camera_left); // create right camera -------------------------------------------- if (use_stereo) { rgb_camera_right = std::make_shared(); // load camera settings std::vector t_BC_vec_r = cfg["rgb_camera_right"]["t_BC"].as>(); std::vector r_BC_vec_r = cfg["rgb_camera_right"]["r_BC"].as>(); Vector<3> t_BC_r(t_BC_vec_r.data()); Matrix<3, 3> r_BC_r = (AngleAxis(r_BC_vec[0] * M_PI / 180.0, Vector<3>::UnitX()) * AngleAxis(r_BC_vec[1] * M_PI / 180.0, Vector<3>::UnitY()) * AngleAxis(r_BC_vec[2] * M_PI / 180.0, Vector<3>::UnitZ())) .toRotationMatrix(); rgb_camera_right->setFOV(flightmare_fov); rgb_camera_right->setWidth(cfg["rgb_camera_left"]["width"].as()); rgb_camera_right->setHeight(cfg["rgb_camera_left"]["height"].as()); rgb_camera_right->setRelPose(t_BC_r, r_BC_r); rgb_camera_right->enableOpticalFlow(false); rgb_camera_right->enableSegmentation(false); rgb_camera_right->enableDepth(false); // add camera to the quadrotor quad_ptr_->addRGBCamera(rgb_camera_right); stereo_baseline_ = fabs(t_BC(1) - t_BC_r(1)); } return true; } } // namespace flightros