@inproceedings{wang2020visual,
  author = {Wang, Chen and Wang, Wenshan and Qiu, Yuheng and Hu, Yafei and Scherer, Sebastian},
  booktitle = {European Conference on Computer Vision (ECCV) },
  keywords = {Unsupervised, Online, Memorability, Interestingness},
  title = {{Visual Memorability for Robotic Interestingness via Unsupervised Online Learning}},
  url = {https://arxiv.org/abs/2005.08829},
  publisher = {Springer},
  year = {2020}
}

In this paper, we aim to solve the problem of interesting scene prediction for mobile robots. This area is currently under explored but is crucial for many practical applications such as autonomous exploration and decision making. First, we expect a robot to detect novel and interesting scenes in unknown environments and lose interests over time after repeatedly observing similar objects. Second, we expect the robots to learn from unbalanced data in a short time, as the robots normally only know the uninteresting scenes before they are deployed. Inspired by those industrial demands, we first propose a novel translation-invariant visual memory for recalling and identifying interesting scenes, then design a three-stage architecture of long-term, short-term, and online learning for human-like experience, environmental knowledge, and online adaption, respectively. It is demonstrated that our approach is able to learn online and find interesting scenes for practical exploration tasks. It also achieves a much higher accuracy than the state-of-the-art algorithm on very challenging robotic interestingness prediction datasets.

@inproceedings{Zhen-2018-109998,
  archiveprefix = {arXiv},
  arxivid = {1810.12515},
  author = {Zhen, Weikun and Scherer, Sebastian},
  booktitle = {International Symposium on Experimental Robotics},
  doi = {10.1007/978-3-030-33950-0_60},
  eprint = {1810.12515},
  month = nov,
  pages = {702--711},
  publisher = {Springer},
  title = {{A Unified 3D Mapping Framework Using a 3D or 2D LiDAR}},
  year = {2020}
}

Simultaneous Localization and Mapping (SLAM) has been considered as a solved problem thanks to the progress made in the past few years. However, the great majority of LiDAR-based SLAM algorithms are designed for a specific type of payload and therefore don’t generalize across different platforms. In practice, this drawback causes the development, deployment and maintenance of an algorithm difficult. Consequently, our work focuses on improving the compatibility across different sensing payloads. Specifically, we extend the Cartographer SLAM library to handle different types of LiDAR including fixed or rotating, 2D or 3D LiDARs. By replacing the localization module of Cartographer and maintaining the sparse pose graph (SPG), the proposed framework can create high-quality 3D maps in real-time on different sensing payloads. Additionally, it brings the benefit of simplicity with only a few parameters need to be adjusted for each sensor type.

@article{Bonatti:2019wb,
  author = {Bonatti, Rogerio and Wang, Wenshan and Ho, Cherie and Ahuja, Aayush and Gschwindt, Mirko and Camci, Efe and Kayacan, Erdal and Choudhury, Sanjiban and Scherer, Sebastian},
  doi = {10.1002/rob.21931},
  issn = {15564967},
  journal = {Journal of Field Robotics},
  keywords = {aerial robotics,cinematography,computer vision,learning,mapping,motion planning},
  month = jan,
  title = {{Autonomous aerial cinematography in unstructured environments with learned artistic decision-making}},
  year = {2020}
}

Aerial cinematography is revolutionizing industries that require live and dynamic camera viewpoints such as entertainment, sports, and security. However, safely piloting a drone while filming a moving target in the presence of obstacles is immensely taxing, often requiring multiple expert human operators. Hence, there is a demand for an autonomous cinematographer that can reason about both geometry and scene context in real-time. Existing approaches do not address all aspects of this problem; they either require high-precision motion-capture systems or global positioning system tags to localize targets, rely on prior maps of the environment, plan for short time horizons, or only follow fixed artistic guidelines specified before the flight. In this study, we address the problem in its entirety and propose a complete system for real-time aerial cinematography that for the first time combines: (a) vision-based target estimation; (b) 3D signed-distance mapping for occlusion estimation; (c) efficient trajectory optimization for long time-horizon camera motion; and (d) learning-based artistic shot selection. We extensively evaluate our system both in simulation and in field experiments by filming dynamic targets moving through unstructured environments. Our results indicate that our system can operate reliably in the real world without restrictive assumptions. We also provide in-depth analysis and discussions for each module, with the hope that our design tradeoffs can generalize to other related applications. Videos of the complete system can be found at https://youtu.be/ookhHnqmlaU.

@inproceedings{Bonatti-2018-111051,
  archiveprefix = {arXiv},
  arxivid = {1808.09563},
  author = {Bonatti, Rogerio and Zhang, Yanfu and Choudhury, Sanjiban and Wang, Wenshan and Scherer, Sebastian},
  booktitle = {International Symposium on Experimental Robotics},
  doi = {10.1007/978-3-030-33950-0_11},
  eprint = {1808.09563},
  month = nov,
  pages = {119--129},
  publisher = {Springer},
  title = {{Autonomous Drone Cinematographer: Using Artistic Principles to Create Smooth, Safe, Occlusion-Free Trajectories for Aerial Filming}},
  year = {2020}
}

Autonomous aerial cinematography has the potential to enable automatic capture of aesthetically pleasing videos without requiring human intervention, empowering individuals with the capability of high-end film studios. Current approaches either only handle off-line trajectory generation, or offer strategies that reason over short time horizons and simplistic representations for obstacles, which result in jerky movement and low real-life applicability. In this work we develop a method for aerial filming that is able to trade off shot smoothness, occlusion, and cinematography guidelines in a principled manner, even under noisy actor predictions. We present a novel algorithm for real-time covariant gradient descent that we use to efficiently find the desired trajectories by optimizing a set of cost functions. Experimental results show that our approach creates attractive shots, avoiding obstacles and occlusion 65 times over 1.25 hours of flight time, re-planning at 5 Hz with a 10 s time horizon. We robustly film human actors, cars and bicycles performing different motion among obstacles, using various shot types.

@article{yu2020monocular,
  archiveprefix = {arXiv},
  arxivid = {2004.00740},
  author = {Yu, Huai and Zhen, Weikun and Yang, Wen and Zhang, Ji and Scherer, Sebastian},
  eprint = {2004.00740},
  journal = {arXiv.org},
  month = apr,
  title = {{Monocular Camera Localization in Prior LiDAR Maps with 2D-3D Line Correspondences}},
  url = {http://arxiv.org/abs/2004.00740},
  year = {2020}
}

Light-weight camera localization in existing maps is essential for vision-based navigation. Currently, visual and visual-inertial odometry (VO\backslash&VIO) techniques are well-developed for state estimation but with inevitable accumulated drifts and pose jumps upon loop closure. To overcome these problems, we propose an efficient monocular camera localization method in prior LiDAR maps using directly estimated 2D-3D line correspondences. To handle the appearance differences and modality gaps between untextured point clouds and images, geometric 3D lines are extracted offline from LiDAR maps while robust 2D lines are extracted online from video sequences. With the pose prediction from VIO, we can efficiently obtain coarse 2D-3D line correspondences. After that, the camera poses and 2D-3D correspondences are iteratively optimized by minimizing the projection error of correspondences and rejecting outliers. The experiment results on the EurocMav dataset and our collected dataset demonstrate that the proposed method can efficiently estimate camera poses without accumulated drifts or pose jumps in urban environments. The code and our collected data are available at https://github.com/levenberg/2D-3D-pose-tracking.

@inproceedings{Pluckter-2018-110919,
  author = {Pluckter, Kevin and Scherer, Sebastian},
  booktitle = {International Symposium on Experimental Robotics},
  doi = {10.1007/978-3-030-33950-0_16},
  month = nov,
  pages = {177--187},
  publisher = {Springer},
  title = {{Precision UAV Landing in Unstructured Environments}},
  year = {2020}
}

Autonomous landing of a drone is a necessary part of autonomous flight. One way to have high certainty of safety in landing is to return to the same location the drone took-off from. Implementations of return-to-home functionality fall short when relying solely on GPS or odometry as inaccuracies in the measurements and drift in the state estimate guides the drone to a position with a large offset from the initial position. This can be particularly dangerous if the drone took-off next to something like a body of water. Current work on precision landing relies on localizing to a known landing pattern, which requires the pilot to carry a landing pattern with them. We propose a method using a downward facing fisheye lens camera to accurately land a UAV from where it took off on an unstructured surface, without a landing pattern. Specifically, this approach uses a position estimate relative to the take-off path of the drone to guide the drone back. With the large Field-of-View provided by the fisheye lens, our algorithm can provide visual feedback starting with a large position error at the beginning of the landing, until 25cm above the ground at the end of the landing. This algorithm empirically shows it can correct the drift error in the state estimation and land with an accuracy of 40cm.

@article{wang2020tartanair,
  archiveprefix = {arXiv},
  arxivid = {2003.14338},
  author = {Wang, Wenshan and Zhu, Delong and Wang, Xiangwei and Hu, Yaoyu and Qiu, Yuheng and Wang, Chen and Hu, Yafei and Kapoor, Ashish and Scherer, Sebastian},
  eprint = {2003.14338},
  journal = {arXiv.org},
  booktitle = {IEEE International Conference on Intelligent Robots and Systems},
  publisher = {IEEE},
  month = mar,
  title = {{TartanAir: A Dataset to Push the Limits of Visual SLAM}},
  url = {http://arxiv.org/abs/2003.14338},
  year = {2020}
}

We present a challenging dataset, the TartanAir, for robot navigation task and more. The data is collected in photo-realistic simulation environments in the presence of various light conditions, weather and moving objects. By collecting data in simulation, we are able to obtain multi-modal sensor data and precise ground truth labels, including the stereo RGB image, depth image, segmentation, optical flow, camera poses, and LiDAR point cloud. We set up a large number of environments with various styles and scenes, covering challenging viewpoints and diverse motion patterns, which are difficult to achieve by using physical data collection platforms.

@article{Zhen:2019fv,
  archiveprefix = {arXiv},
  arxivid = {1907.00930},
  author = {Zhen, Weikun and Hu, Yaoyu and Liu, Jingfeng and Scherer, Sebastian},
  doi = {10.1109/LRA.2019.2928261},
  eprint = {1907.00930},
  issn = {23773766},
  journal = {IEEE Robotics and Automation Letters},
  keywords = {Calibration and Identification,Mapping,Sensor Fusion},
  month = oct,
  number = {4},
  pages = {3585--3592},
  title = {{A Joint Optimization Approach of LiDAR-Camera Fusion for Accurate Dense 3-D Reconstructions}},
  volume = {4},
  year = {2019}
}

Fusing data from LiDAR and camera is conceptually attractive because of their complementary properties. For instance, camera images are of higher resolution and have colors, while LiDAR data provide more accurate range measurements and have a wider field of view. However, the sensor fusion problem remains challenging since it is difficult to find reliable correlations between data of very different characteristics (geometry versus texture, sparse versus dense). This letter proposes an offline LiDAR-camera fusion method to build dense, accurate 3-D models. Specifically, our method jointly solves a bundle adjustment problem and a cloud registration problem to compute camera poses and the sensor extrinsic calibration. In experiments, we show that our method can achieve an average accuracy of 2.7 mm and resolution of 70 points/cm2 by comparing to the ground truth data from a survey scanner. Furthermore, the extrinsic calibration result is discussed and shown to outperform the state-of-the-art method.

@article{Keller:2019uu,
  annote = {6 pages, 5 figures},
  archiveprefix = {arXiv},
  arxivid = {1910.04874},
  author = {Keller, John and Scherer, Sebastian},
  eprint = {1910.04874},
  journal = {arXiv.org},
  month = oct,
  title = {{A Stereo Algorithm for Thin Obstacles and Reflective Objects}},
  url = {http://arxiv.org/abs/1910.04874},
  year = {2019}
}

Stereo cameras are a popular choice for obstacle avoidance for outdoor lighweight, low-cost robotics applications. However, they are unable to sense thin and reflective objects well. Currently, many algorithms are tuned to perform well on indoor scenes like the Middlebury dataset. When navigating outdoors, reflective objects, like windows and glass, and thin obstacles, like wires, are not well handled by most stereo disparity algorithms. Reflections, repeating patterns and objects parallel to the cameras’ baseline causes mismatches between image pairs which leads to bad disparity estimates. Thin obstacles are difficult for many sliding window based disparity methods to detect because they do not take up large portions of the pixels in the sliding window. We use a trinocular camera setup and micropolarizer camera capable of detecting reflective objects to overcome these issues. We present a hierarchical disparity algorithm that reduces noise, separately identify wires using semantic object triangulation in three images, and use information about the polarization of light to estimate the disparity of reflective objects. We evaluate our approach on outdoor data that we collected. Our method contained an average of 9.27% of bad pixels compared to a typical stereo algorithm’s 18.4% of bad pixels in scenes containing reflective objects. Our trinocular and semantic wire disparity methods detected 53% of wire pixels, whereas a typical two camera stereo algorithm detected 5%.

@article{Scherer:2019ub,
  author = {Scherer, Sebastian},
  month = aug,
  pages = {1--11},
  title = {{Aerial Contact Sensing for Improved Inspection of Transportation Infrastructure}},
  year = {2019}
}

@article{Keipour:2019uy,
  archiveprefix = {arXiv},
  arxivid = {1907.06268},
  author = {Keipour, Azarakhsh and Mousaei, Mohammadreza and Scherer, Sebastian},
  eprint = {1907.06268},
  month = jul,
  title = {{ALFA: A Dataset for UAV Fault and Anomaly Detection}},
  url = {http://arxiv.org/abs/1907.06268},
  year = {2019}
}

We present a dataset of several fault types in control surfaces of a fixed-wing Unmanned Aerial Vehicle (UAV) for use in Fault Detection and Isolation (FDI) and Anomaly Detection (AD) research. Currently, the dataset includes processed data for 47 autonomous flights with scenarios for eight different types of control surface (actuator and engine) faults, with a total of 66 minutes of flight in normal conditions and 13 minutes of post-fault flight time. It additionally includes many hours of raw data of fully-autonomous, autopilot-assisted and manual flights with tens of fault scenarios. The ground truth of the time and type of faults is provided in each scenario to enable evaluation of the methods using the dataset. We have also provided the helper tools in several programming languages to load and work with the data and to help the evaluation of a detection method using the dataset. A set of metrics is proposed to help to compare different methods using the dataset. Most of the current fault detection methods are evaluated in simulation and as far as we know, this dataset is the only one providing the real flight data with faults in such capacity. We hope it will help advance the state-of-the-art in Anomaly Detection or FDI research for Autonomous Aerial Vehicles and mobile robots to enhance the safety of autonomous and remote flight operations further. The dataset and the provided tools can be accessed from http://theairlab.org/alfa-dataset.

@article{Keipour:2019ig,
  annote = {2019 IEEE International Conference on Robotics and Automation (ICRA) [7 pages, 7 figures]},
  archiveprefix = {arXiv},
  arxivid = {1907.00511},
  author = {Keipour, Azarakhsh and Mousaei, Mohammadreza and Scherer, Sebastian},
  doi = {10.1109/ICRA.2019.8794286},
  eprint = {1907.00511},
  isbn = {9781538660263},
  issn = {10504729},
  journal = {Proceedings - IEEE International Conference on Robotics and Automation},
  month = jun,
  pages = {5679--5685},
  title = {{Automatic real-time anomaly detection for autonomous aerial vehicles}},
  volume = {2019-May},
  year = {2019}
}

The recent increase in the use of aerial vehicles raises concerns about the safety and reliability of autonomous operations. There is a growing need for methods to monitor the status of these aircraft and report any faults and anomalies to the safety pilot or to the autopilot to deal with the emergency situation. In this paper, we present a real-time approach using the Recursive Least Squares method to detect anomalies in the behavior of an aircraft. The method models the relationship between correlated input-output pairs online and uses the model to detect the anomalies. The result is an easy-to-deploy anomaly detection method that does not assume a specific aircraft model and can detect many types of faults and anomalies in a wide range of autonomous aircraft. The experiments on this method show a precision of 88.23%, recall of 88. 23% and 86. 36% accuracy for over 22 flight tests. The other contribution is providing a new fault detection open dataset for autonomous aircraft, which contains complete data and the ground truth for 22 fixed-wing flights with eight different types of mid-flight actuator failures to help future fault detection research for aircraft.

@article{Yang:2018eo,
  annote = {IEEE Transactions on Robotics},
  archiveprefix = {arXiv},
  arxivid = {1806.00557},
  author = {Yang, Shichao and Scherer, Sebastian},
  doi = {10.1109/TRO.2019.2909168},
  eprint = {1806.00557},
  issn = {19410468},
  journal = {IEEE Transactions on Robotics},
  keywords = {Dynamic SLAM,SLAM,object detection,object simultaneous localization and mapping (SLAM)},
  month = jun,
  number = {4},
  pages = {925--938},
  title = {{CubeSLAM: Monocular 3-D Object SLAM}},
  volume = {35},
  year = {2019}
}

In this paper, we present a method for single image three-dimensional (3-D) cuboid object detection and multiview object simultaneous localization and mapping in both static and dynamic environments, and demonstrate that the two parts can improve each other. First, for single image object detection, we generate high-quality cuboid proposals from two-dimensional (2-D) bounding boxes and vanishing points sampling. The proposals are further scored and selected based on the alignment with image edges. Second, multiview bundle adjustment with new object measurements is proposed to jointly optimize poses of cameras, objects, and points. Objects can provide long-range geometric and scale constraints to improve camera pose estimation and reduce monocular drift. Instead of treating dynamic regions as outliers, we utilize object representation and motion model constraints to improve the camera pose estimation. The 3-D detection experiments on SUN RGBD and KITTI show better accuracy and robustness over existing approaches. On the public TUM, KITTI odometry and our own collected datasets, our SLAM method achieves the state-of-the-art monocular camera pose estimation and at the same time, improves the 3-D object detection accuracy.

@inproceedings{Chandarana-2018-107474,
  author = {Chandarana, Meghan and Luo, Wenhao and Lewis, Michael and Sycara, Katia and Scherer, Sebastian},
  booktitle = {Proceedings - 2018 IEEE International Conference on Systems, Man, and Cybernetics, SMC 2018},
  doi = {10.1109/SMC.2018.00212},
  isbn = {9781538666500},
  month = oct,
  pages = {1209--1214},
  title = {{Decentralized Method for Sub-Swarm Deployment and Rejoining}},
  year = {2019}
}

As part of swarm search and service (SSS) missions, robots are tasked with servicing jobs as they are sensed. This requires small sub-swarm teams to leave the swarm for a specified amount of time to service the jobs. In doing so, fewer robots are required to change motion than if the whole swarm were diverted, thereby minimizing the job’s overall effect on the swarm’s main goal. We explore the problem of removing the required number of robots from the swarm, while maintaining overall swarm connectivity. By preserving connectivity, robots are able to successfully rejoin the swarm upon completion of their assigned job. These robots are then made available for reallocation. We propose a decentralized and asynchronous method for breaking off sub-swarm groups and rejoining them with the main swarm using the swarm’s communication graph topology. Both single and multiple job site cases are explored. The results are compared against a full swarm movement method. Simulation results show that the proposed method outperforms a full swarm method in the average number of messages sent per robot in each step, as well as, the distance traveled by the swarm.

@article{Hu:2019vf,
  annote = {Submitted to International Conference on Robotics and Automation (ICRA2020)},
  archiveprefix = {arXiv},
  arxivid = {1912.05012},
  author = {Hu, Yaoyu and Zhen, Weikun and Scherer, Sebastian},
  eprint = {1912.05012},
  journal = {arXiv.org},
  month = nov,
  title = {{Deep-Learning Assisted High-Resolution Binocular Stereo Depth Reconstruction}},
  url = {http://arxiv.org/abs/1912.05012},
  year = {2019}
}

This work presents dense stereo reconstruction using high-resolution images for infrastructure inspections. The state-of-the-art stereo reconstruction methods, both learning and non-learning ones, consume too much computational resource on high-resolution data. Recent learning-based methods achieve top ranks on most benchmarks. However, they suffer from the generalization issue due to lack of task-specific training data. We propose to use a less resource demanding non-learning method, guided by a learning-based model, to handle high-resolution images and achieve accurate stereo reconstruction. The deep-learning model produces an initial disparity prediction with uncertainty for each pixel of the down-sampled stereo image pair. The uncertainty serves as a self-measurement of its generalization ability and the per-pixel searching range around the initially predicted disparity. The downstream process performs a modified version of the Semi-Global Block Matching method with the up-sampled per-pixel searching range. The proposed deep-learning assisted method is evaluated on the Middlebury dataset and high-resolution stereo images collected by our customized binocular stereo camera. The combination of learning and non-learning methods achieves better performance on 12 out of 15 cases of the Middlebury dataset. In our infrastructure inspection experiments, the average 3D reconstruction error is less than 0.004m.

@inproceedings{Zhen:2019jn,
  author = {Zhen, Weikun and Scherer, Sebastian},
  booktitle = {Proceedings - IEEE International Conference on Robotics and Automation},
  doi = {10.1109/ICRA.2019.8794167},
  isbn = {9781538660263},
  issn = {10504729},
  month = may,
  pages = {4903--4908},
  publisher = {IEEE},
  title = {{Estimating the localizability in tunnel-like environments using LiDAR and UWB}},
  volume = {2019-May},
  year = {2019}
}

The application of robots in inspection tasks has been growing quickly thanks to the advancements in autonomous navigation technology, especially the robot localization techniques in GPS-denied environments. Although many methods have been proposed to localize a robot using onboard sensors such as cameras and LiDARs, achieving robust localization in geometrically degenerated environments, e.g. tunnels, remains a challenging problem. In this work, we focus on the robust localization problem in such situations. A novel degeneration characterization model is presented to estimate the localizability at a given location in the prior map. And the localizability of a LiDAR and an Ultra-Wideband (UWB) ranging radio is analyzed. Additionally, a probabilistic sensor fusion method is developed to combine IMU, LiDAR and the UWB. Experiment results show that this method allows for robust localization inside a long straight tunnel.

@article{Choudhury:2019hz,
  author = {Choudhury, Sanjiban and Dugar, Vishal and Maeta, Silvio and MacAllister, Brian and Arora, Sankalp and Althoff, Daniel and Scherer, Sebastian},
  doi = {10.1002/rob.21906},
  issn = {15564967},
  journal = {Journal of Field Robotics},
  keywords = {aerial robotics,learning,planning},
  month = dec,
  number = {8},
  pages = {1275--1332},
  title = {{High performance and safe flight of full-scale helicopters from takeoff to landing with an ensemble of planners}},
  volume = {36},
  year = {2019}
}

Autonomous flight of unmanned full-size rotor-craft has the potential to enable many new applications. However, the dynamics of these aircraft, prevailing wind conditions, the need to operate over a variety of speeds and stringent safety requirements make it difficult to generate safe plans for these systems. Prior work has shown results for only parts of the problem. Here we present the first comprehensive approach to planning safe trajectories for autonomous helicopters from takeoff to landing. Our approach is based on two key insights. First, we compose an approximate solution by cascading various modules that can efficiently solve different relaxations of the planning problem. Our framework invokes a long-term route optimizer, which feeds a receding-horizon planner which in turn feeds a high-fidelity safety executive. Secondly, to deal with the diverse planning scenarios that may arise, we hedge our bets with an ensemble of planners. We use a data-driven approach that maps a planning context to a diverse list of planning algorithms that maximize the likelihood of success. Our approach was extensively evaluated in simulation and in real-world flight tests on three different helicopter systems for duration of more than 109 autonomous hours and 590 pilot-in-the-loop hours. We provide an in-depth analysis and discuss the various tradeoffs of decoupling the problem, using approximations and leveraging statistical techniques. We summarize the insights with the hope that it generalizes to other platforms and applications.

@inproceedings{Wang:2019kd,
  archiveprefix = {arXiv},
  arxivid = {1903.11174},
  author = {Wang, Wenshan and Ahuja, Aayush and Zhang, Yanfu and Bonatti, Rogerio and Scherer, Sebastian},
  booktitle = {Proceedings - IEEE International Conference on Robotics and Automation},
  doi = {10.1109/ICRA.2019.8793994},
  eprint = {1903.11174},
  isbn = {9781538660263},
  issn = {10504729},
  month = may,
  pages = {5901--5907},
  publisher = {IEEE},
  title = {{Improved generalization of heading direction estimation for aerial filming using semi-supervised regression}},
  volume = {2019-May},
  year = {2019}
}

In the task of Autonomous aerial filming of a moving actor (e.g. a person or a vehicle), it is crucial to have a good heading direction estimation for the actor from the visual input. However, the models obtained in other similar tasks, such as pedestrian collision risk analysis and human-robot interaction, are very difficult to generalize to the aerial filming task, because of the difference in data distributions. Towards improving generalization with less amount of labeled data, this paper presents a semi-supervised algorithm for heading direction estimation problem. We utilize temporal continuity as the unsupervised signal to regularize the model and achieve better generalization ability. This semi-supervised algorithm is applied to both training and testing phases, which increases the testing performance by a large margin. We show that by leveraging unlabeled sequences, the amount of labeled data required can be significantly reduced. We also discuss several important details on improving the performance by balancing labeled and unlabeled loss, and making good combinations. Experimental results show that our approach robustly outputs the heading direction for different types of actor. The aesthetic value of the video is also improved in the aerial filming task.

@article{Bonatti:2019vp,
  archiveprefix = {arXiv},
  arxivid = {1909.06993},
  author = {Bonatti, Rogerio and Madaan, Ratnesh and Vineet, Vibhav and Scherer, Sebastian and Kapoor, Ashish},
  eprint = {1909.06993},
  journal = {arXiv:1909.06993v1 [cs.CV]},
  month = sep,
  title = {{Learning Visuomotor Policies for Aerial Navigation Using Cross-Modal Representations}},
  url = {http://arxiv.org/abs/1909.06993},
  year = {2019}
}

Machines are a long way from robustly solving open-world perception-control tasks, such as first-person view (FPV) aerial navigation. While recent advances in end-to-end Machine Learning, especially Imitation and Reinforcement Learning appear promising, they are constrained by the need of large amounts of difficult-to-collect labeled real-world data. Simulated data, on the other hand, is easy to generate, but generally does not render safe behaviors in diverse real-life scenarios. In this work we propose a novel method for learning robust visuomotor policies for real-world deployment which can be trained purely with simulated data. We develop rich state representations that combine supervised and unsupervised environment data. Our approach takes a cross-modal perspective, where separate modalities correspond to the raw camera data and the system states relevant to the task, such as the relative pose of gates to the drone in the case of drone racing. We feed both data modalities into a novel factored architecture, which learns a joint low-dimensional embedding via Variational Auto Encoders. This compact representation is then fed into a control policy, which we trained using imitation learning with expert trajectories in a simulator. We analyze the rich latent spaces learned with our proposed representations, and show that the use of our cross-modal architecture significantly improves control policy performance as compared to end-to-end learning or purely unsupervised feature extractors. We also present real-world results for drone navigation through gates in different track configurations and environmental conditions. Our proposed method, which runs fully onboard, can successfully generalize the learned representations and policies across simulation and reality, significantly outperforming baseline approaches. Supplementary video: https://youtu.be/VKc3A5HlUU8

@article{Zhen:2019up,
  annote = {6 pages plus reference. Work has been submitted to ICRA 2020},
  archiveprefix = {arXiv},
  arxivid = {1911.03369},
  author = {Zhen, Weikun and Hu, Yaoyu and Yu, Huai and Scherer, Sebastian},
  eprint = {1911.03369},
  journal = {arXiv.org},
  month = nov,
  title = {{LiDAR Enhanced Structure-from-Motion}},
  url = {http://arxiv.org/abs/1911.03369},
  year = {2019}
}

Although Structure-from-Motion (SfM) as a maturing technique has been widely used in many applications, state-of-the-art SfM algorithms are still not robust enough in certain situations. For example, images for inspection purposes are often taken in close distance to obtain detailed textures, which will result in less overlap between images and thus decrease the accuracy of estimated motion. In this paper, we propose a LiDAR-enhanced SfM pipeline that jointly processes data from a rotating LiDAR and a stereo camera pair to estimate sensor motions. We show that incorporating LiDAR helps to effectively reject falsely matched images and significantly improve the model consistency in large-scale environments. Experiments are conducted in different environments to test the performance of the proposed pipeline and comparison results with the state-of-the-art SfM algorithms are reported.

@article{Yu:2019ub,
  annote = {8 pages, Work has been submitted to ICRA 2020},
  archiveprefix = {arXiv},
  arxivid = {1912.05013},
  author = {Yu, Huai and Zhen, Weikun and Yang, Wen and Scherer, Sebastian},
  eprint = {1912.05013},
  journal = {arXiv.org},
  month = nov,
  title = {{Line-based Camera Pose Estimation in Point Cloud of Structured Environments}},
  url = {http://arxiv.org/abs/1912.05013},
  year = {2019}
}

Accurate registration of 2D imagery with point clouds is a key technology for image-LiDAR point cloud fusion, camera to laser scanner calibration and camera localization. Despite continuous improvements, automatic registration of 2D and 3D data without using additional textured information still faces great challenges. In this paper, we propose a new 2D-3D registration method to estimate 2D-3D line feature correspondences and the camera pose in untextured point clouds of structured environments. Specifically, we first use geometric constraints between vanishing points and 3D parallel lines to compute all feasible camera rotations. Then, we utilize a hypothesis testing strategy to estimate the 2D-3D line correspondences and the translation vector. By checking the consistency with computed correspondences, the best rotation matrix can be found. Finally, the camera pose is further refined using non-linear optimization with all the 2D-3D line correspondences. The experimental results demonstrate the effectiveness of the proposed method on the synthetic and real dataset (outdoors and indoors) with repeated structures and rapid depth changes.

@article{Yang:2019el,
  author = {Yang, Shichao and Scherer, Sebastian},
  doi = {10.1109/LRA.2019.2924848},
  issn = {23773766},
  journal = {IEEE Robotics and Automation Letters},
  keywords = {Object and plane slam,Semantic scene understanding,Slam},
  number = {4},
  pages = {3145--3152},
  title = {{Monocular object and plane slam in structured environments}},
  volume = {4},
  year = {2019}
}

In this letter, we present a monocular simultaneous localization and mapping (SLAM) algorithm using high-level object and plane landmarks. The built map is denser, more compact and semantic meaningful compared to feature point based SLAM. We first propose a high-order graphical model to jointly infer the three-dimensional object and layout planes from single images considering occlusions and semantic constraints. The extracted objects and planes are further optimized with camera poses in a unified SLAM framework. Objects and planes can provide more semantic constraints such as Manhattan plane and object supporting relationships compared to points. Experiments on various public and collected datasets, including ICL NUIM and TUM Mono show that our algorithm can improve camera localization accuracy compared to state-of-the-art SLAM, especially when there is no loop closure, and also generate dense maps robustly in many structured environments.

@inproceedings{Madaan:2019kb,
  author = {Madaan, Ratnesh and Kaess, Michael and Scherer, Sebastian},
  booktitle = {Proceedings - IEEE International Conference on Robotics and Automation},
  doi = {10.1109/ICRA.2019.8793852},
  isbn = {9781538660263},
  issn = {10504729},
  month = may,
  pages = {5657--5664},
  publisher = {IEEE},
  title = {{Multi-view reconstruction of wires using a catenary model}},
  volume = {2019-May},
  year = {2019}
}

Reliable detection and reconstruction of wires is one of the hardest problems in the UAV community, with a wide ranging impact in the industry in terms of wire avoidance capabilities and powerline corridor inspection. In this work, we introduce a real-time, model-based, multi-view algorithm to reconstruct wires from a set of images with known camera poses, while exploiting their natural shape-the catenary curve. Using a model-based approach helps us deal with partial wire detections in images, which may occur due to natural occlusion and false negatives. In addition, using a parsimonious model makes our algorithm efficient as we only need to optimize for 5 model parameters, as opposed to hundreds of 3D points in bundle-adjustment approaches. Our algorithm obviates the need for pixel correspondences by computing the reprojection error via the distance transform of binarized wire segmentation images. Further, we make our algorithm robust to arbitrary initializations by introducing an on-demand, approximate extrapolation of the distance transform based objective. We demonstrate the effectiveness of our algorithm against false negatives and random initializations in simulation, and show qualitative results with real data collected from a small UAV.

@inproceedings{Zhen:2018uf,
  address = {Pittsburgh, Pa},
  author = {Zhen, Weikun and Scherer, Sebastian},
  booktitle = {Workshop on Challenges and Opportunities for Resilient Collective Intelligence in Subterranean Environments},
  month = jun,
  title = {{Achieving Robust Localization in Geometrically Degenerated Tunnels}},
  year = {2018}
}

Although many methods have been proposed to localize a robot using onboard sensors in GPS-denied environments , achieving robust localization in geometrically degenerated tunnels remains a challenging problem in robot-based inspection tasks. In this work, we first present a novel model to analyze the localizability of the prior map at a given location. Then we propose the utilization of a single Ultra-Wideband (UWB) ranging device to compensate for the degeneration of LiDAR based localization inside tunnels. A probabilistic sensor fusion method is developed and demonstrated to achieve real-time robust localization inside a geometrically degenerated tunnel.

@article{Dong:2018cg,
  author = {Dong, Zhen and Yang, Bisheng and Hu, Pingbo and Scherer, Sebastian},
  doi = {10.1016/j.isprsjprs.2018.01.013},
  issn = {09242716},
  journal = {ISPRS Journal of Photogrammetry and Remote Sensing},
  keywords = {Energy optimization,Guided sampling,Hybrid region growing,Multiscale supervoxel,Plane segmentation,Simulated annealing},
  month = mar,
  pages = {112--133},
  title = {{An efficient global energy optimization approach for robust 3D plane segmentation of point clouds}},
  volume = {137},
  year = {2018}
}

Automatic 3D plane segmentation is necessary for many applications including point cloud registration, building information model (BIM) reconstruction, simultaneous localization and mapping (SLAM), and point cloud compression. However, most of the existing 3D plane segmentation methods still suffer from low precision and recall, and inaccurate and incomplete boundaries, especially for low-quality point clouds collected by RGB-D sensors. To overcome these challenges, this paper formulates the plane segmentation problem as a global energy optimization because it is robust to high levels of noise and clutter. First, the proposed method divides the raw point cloud into multiscale supervoxels, and considers planar supervoxels and individual points corresponding to nonplanar supervoxels as basic units. Then, an efficient hybrid region growing algorithm is utilized to generate initial plane set by incrementally merging adjacent basic units with similar features. Next, the initial plane set is further enriched and refined in a mutually reinforcing manner under the framework of global energy optimization. Finally, the performances of the proposed method are evaluated with respect to six metrics (i.e., plane precision, plane recall, under-segmentation rate, over-segmentation rate, boundary precision, and boundary recall) on two benchmark datasets. Comprehensive experiments demonstrate that the proposed method obtained good performances both in high-quality TLS point clouds (i.e., SEMANTIC3D.NET dataset) and low-quality RGB-D point clouds (i.e., S3DIS dataset) with six metrics of (94.2%, 95.1%, 2.9%, 3.8%, 93.6%, 94.1%) and (90.4%, 91.4%, 8.2%, 7.6%, 90.8%, 91.7%) respectively.

@article{Choudhury:2017wf,
  archiveprefix = {arXiv},
  arxivid = {1711.07329},
  author = {Choudhury, Sanjiban and Srinivasa, Siddhartha and Scherer, Sebastian},
  doi = {10.24963/ijcai.2018/679},
  eprint = {1711.07329},
  isbn = {9780999241127},
  issn = {10450823},
  journal = {IJCAI International Joint Conference on Artificial Intelligence},
  month = nov,
  pages = {4890--4897},
  title = {{Bayesian active edge evaluation on expensive graphs}},
  volume = {2018-July},
  year = {2018}
}

We consider the problem of real-time motion planning that requires evaluating a minimal number of edges on a graph to quickly discover collision-free paths. Evaluating edges is expensive, both for robots with complex geometries like robot arms, and for robots sensing the world online like UAVs. Until now, this challenge has been addressed via laziness i.e. deferring edge evaluation until absolutely necessary, with the hope that edges turn out to be valid. However, all edges are not alike in value - some have a lot of potentially good paths flowing through them, and some others encode the likelihood of neighbouring edges being valid. This leads to our key insight - instead of passive laziness, we can actively choose edges that reduce the uncertainty about the validity of paths. We show that this is equivalent to the Bayesian active learning paradigm of decision region determination (DRD). However, the DRD problem is not only combina-torially hard, but also requires explicit enumeration of all possible worlds. We propose a novel framework that combines two DRD algorithms, DIRECT and BISECT, to overcome both issues. We show that our approach outperforms several state-of-the-art algorithms on a spectrum of planning problems for mobile robots, manipulators and autonomous helicopters.

@article{Choudhury:2017vh,
  archiveprefix = {arXiv},
  arxivid = {1711.06391},
  author = {Choudhury, Sanjiban and Bhardwaj, Mohak and Arora, Sankalp and Kapoor, Ashish and Ranade, Gireeja and Scherer, Sebastian and Dey, Debadeepta},
  doi = {10.1177/0278364918781001},
  eprint = {1711.06391},
  issn = {17413176},
  journal = {International Journal of Robotics Research},
  keywords = {,Imitation learning,POMDPs,QMDPs,heuristic search,sequential decision making},
  month = dec,
  number = {13-14},
  pages = {1632--1672},
  title = {{Data-driven planning via imitation learning}},
  volume = {37},
  year = {2018}
}

Robot planning is the process of selecting a sequence of actions that optimize for a task=specific objective. For instance, the objective for a navigation task would be to find collision-free paths, whereas the objective for an exploration task would be to map unknown areas. The optimal solutions to such tasks are heavily influenced by the implicit structure in the environment, i.e. the configuration of objects in the world. State-of-the-art planning approaches, however, do not exploit this structure, thereby expending valuable effort searching the action space instead of focusing on potentially good actions. In this paper, we address the problem of enabling planners to adapt their search strategies by inferring such good actions in an efficient manner using only the information uncovered by the search up until that time. We formulate this as a problem of sequential decision making under uncertainty where at a given iteration a planning policy must map the state of the search to a planning action. Unfortunately, the training process for such partial-information-based policies is slow to converge and susceptible to poor local minima. Our key insight is that if we could fully observe the underlying world map, we would easily be able to disambiguate between good and bad actions. We hence present a novel data-driven imitation learning framework to efficiently train planning policies by imitating a clairvoyant oracle: an oracle that at train time has full knowledge about the world map and can compute optimal decisions. We leverage the fact that for planning problems, such oracles can be efficiently computed and derive performance guarantees for the learnt policy. We examine two important domains that rely on partial-information-based policies: informative path planning and search-based motion planning. We validate the approach on a spectrum of environments for both problem domains, including experiments on a real UAV, and show that the learnt policy consistently outperforms state-of-the-art algorithms. Our framework is able to train policies that achieve up to 39% more reward than state-of-the art information-gathering heuristics and a 70 x speedup as compared with A* on search-based planning problems. Our approach paves the way forward for applying data-driven techniques to other such problem domains under the umbrella of robot planning.

@inproceedings{Chandarana-2018-107676,
  author = {Chandarana, Meghan and Lewis, Michael and Sycara, Katia and Scherer, Sebastian},
  booktitle = {IEEE International Conference on Intelligent Robots and Systems},
  doi = {10.1109/IROS.2018.8593919},
  isbn = {9781538680940},
  issn = {21530866},
  month = sep,
  pages = {4848--4853},
  title = {{Determining Effective Swarm Sizes for Multi-Job Type Missions}},
  year = {2018}
}

Swarm search and service (SSS) missions require large swarms to simultaneously search an area while servicing jobs as they are encountered. Jobs must be immediately serviced and can be one of several different job types - each requiring a different service time and number of vehicles to complete its service successfully. After jobs are serviced, vehicles are returned to the swarm and become available for reallocation. As part of SSS mission planning, human operators must determine the number of vehicles needed to achieve this balance. The complexities associated with balancing vehicle allocation to multiple as yet unknown tasks with returning vehicles makes this extremely difficult for humans. Previous work assumes that all system jobs are known ahead of time or that vehicles move independently of each other in a multi-agent framework. We present a dynamic vehicle routing (DVR) framework whose policies optimally allocate vehicles as jobs arrive. By incorporating time constraints into the DVR framework, an M/M/k/k queuing model can be used to evaluate overall steady state system performance for a given swarm size. Using these estimates, operators can rapidly compare system performance across different configurations, leading to more effective choices for swarm size. A sensitivity analysis is performed and its results are compared with the model, illustrating the appropriateness of our method to problems of plausible scale and complexity.

@inproceedings{Dubey-2018-107515,
  author = {Dubey, Geetesh and Madaan, Ratnesh and Scherer, Sebastian},
  booktitle = {IEEE International Conference on Intelligent Robots and Systems},
  doi = {10.1109/IROS.2018.8593499},
  isbn = {9781538680940},
  issn = {21530866},
  month = oct,
  pages = {6311--6318},
  title = {{DROAN - Disparity-Space Representation for Obstacle Avoidance: Enabling Wire Mapping Avoidance}},
  year = {2018}
}

Wire detection, depth estimation and avoidance is one of the hardest challenges towards the ubiquitous presence of robust autonomous aerial vehicles. We present an approach and a system which tackles these three challenges along with generic obstacle avoidance as well. First, we perform monocular wire detection using a convolutional neural network under the semantic segmentation paradigm, and obtain a confidence map of wire pixels. Along with this, we also use a binocular stereo pair to detect other generic obstacles. We represent wires and generic obstacles using a disparity space representation and do a C-space expansion by using a non-linear sensor model we develop. Occupancy inference for collision checking is performed by maintaining a pose graph over multiple disparity images. For avoidance of wire and generic obstacles, we use a precomputed trajectory library, which is evaluated in an online fashion in accordance to a cost function over proximity to the goal. We follow this trajectory with a path tracking controller. Finally, we demonstrate the effectiveness of our proposed method in simulation for wire mapping, and on hardware by multiple runs for both wire and generic obstacle avoidance.

@article{Dong-2018-106594,
  author = {Dong, Zhen and Yang, Bisheng and Liang, Fuxun and Huang, Ronggang and Scherer, Sebastian},
  doi = {10.1016/j.isprsjprs.2018.06.018},
  issn = {09242716},
  journal = {ISPRS Journal of Photogrammetry and Remote Sensing},
  keywords = {Binary shape context,Hierarchical registration,Multiple overlaps,Point cloud registration,Point cloud similarity,Vector of locally aggregated descriptors},
  month = oct,
  pages = {61--79},
  title = {{Hierarchical registration of unordered TLS point clouds based on binary shape context descriptor}},
  volume = {144},
  year = {2018}
}

Automatic registration of unordered point clouds collected by the terrestrial laser scanner (TLS) is the prerequisite for many applications including 3D model reconstruction, cultural heritage management, forest structure assessment, landslide monitoring, and solar energy analysis. However, most of the existing point cloud registration methods still suffer from some limitations. On one hand, most of them are considerable time-consuming and high computational complexity due to the exhaustive pairwise search for recovering the underlying overlaps, which makes them infeasible for the registration of large-scale point clouds. On the other hand, most of them only leverage pairwise overlaps and rarely use the overlaps between multiple point clouds, resulting in difficulty dealing with point clouds with limited overlaps. To overcome these limitations, this paper presents a Hierarchical Merging based Multiview Registration (HMMR) algorithm to align unordered point clouds from various scenes. First, the multi-level descriptors (i.e., local descriptor: Binary Shape Context (BSC) and global descriptor: Vector of Locally Aggregated Descriptor (VLAD)) are calculated. Second, the point clouds overlapping (adjacent) graph is efficiently constructed by leveraging the similarity between their corresponding VLAD vectors. Finally, the proposed method hierarchically registers multiple point clouds by iteratively performing optimal registration point clouds calculation, BSC descriptor based pairwise registration and point cloud groups overlapping (adjacent) graph update, until all the point clouds are aligned into a common coordinate reference. Comprehensive experiments demonstrate that the proposed algorithm obtains good performance in terms of successful registration rate, rotation error, translation error, and runtime, and outperformed the state-of-the-art approaches.

@article{Arora:2018vy,
  annote = {6 pages, 1 figure},
  archiveprefix = {arXiv},
  arxivid = {1804.02573},
  author = {Arora, Sankalp and Choudhury, Sanjiban and Scherer, Sebastian},
  eprint = {1804.02573},
  journal = {arXiv.org},
  month = apr,
  title = {{Hindsight is Only 50/50: Unsuitability of MDP based Approximate POMDP Solvers for Multi-resolution Information Gathering}},
  url = {http://arxiv.org/abs/1804.02573},
  year = {2018}
}

Partially Observable Markov Decision Processes (POMDPs) offer an elegant framework to model sequential decision making in uncertain environments. Solving POMDPs online is an active area of research and given the size of real-world problems approximate solvers are used. Recently, a few approaches have been suggested for solving POMDPs by using MDP solvers in conjunction with imitation learning. MDP based POMDP solvers work well for some cases, while catastrophically failing for others. The main failure point of such solvers is the lack of motivation for MDP solvers to gain information, since under their assumption the environment is either already known as much as it can be or the uncertainty will disappear after the next step. However for solving POMDP problems gaining information can lead to efficient solutions. In this paper we derive a set of conditions where MDP based POMDP solvers are provably sub-optimal. We then use the well-known tiger problem to demonstrate such sub-optimality. We show that multi-resolution, budgeted information gathering cannot be addressed using MDP based POMDP solvers. The contribution of the paper helps identify the properties of a POMDP problem for which the use of MDP based POMDP solvers is inappropriate, enabling better design choices.

@inproceedings{yanfu-2018-111053,
  archiveprefix = {arXiv},
  arxivid = {1810.07225},
  author = {Zhang, Yanfu and Wang, Wenshan and Bonatti, Rogerio and Maturana, Daniel and Scherer, Sebastian},
  booktitle = {Conference on Robot Learning},
  eprint = {1810.07225},
  month = oct,
  publisher = {Journal of Machine Learning Research},
  title = {{Integrating kinematics and environment context into deep inverse reinforcement learning for predicting off-road vehicle trajectories}},
  url = {http://arxiv.org/abs/1810.07225},
  year = {2018}
}

Predicting the motion of a mobile agent from a third-person perspective is an important component for many robotics applications, such as autonomous navigation and tracking. With accurate motion prediction of other agents, robots can plan for more intelligent behaviors to achieve specified objectives, instead of acting in a purely reactive way. Previous work addresses motion prediction by either only filtering kinematics, or using hand-designed and learned representations of the environment. Instead of separating kinematic and environmental context, we propose a novel approach to integrate both into an inverse reinforcement learning (IRL) framework for trajectory prediction. Instead of exponentially increasing the state-space complexity with kinematics, we propose a two-stage neural network architecture that considers motion and environment together to recover the reward function. The first-stage network learns feature representations of the environment using low-level LiDAR statistics and the second-stage network combines those learned features with kinematics data. We collected over 30 km of off-road driving data and validated experimentally that our method can effectively extract useful environmental and kinematic features. We generate accurate predictions of the distribution of future trajectories of the vehicle, encoding complex behaviors such as multi-modal distributions at road intersections, and even show different predictions at the same intersection depending on the vehicle’s speed.

@article{Das:2018es,
  archiveprefix = {arXiv},
  arxivid = {1811.02563},
  author = {Das, Manash Pratim and Dong, Zhen and Scherer, Sebastian},
  doi = {10.1109/IROS.2018.8594318},
  eprint = {1811.02563},
  isbn = {9781538680940},
  issn = {21530866},
  journal = {IEEE International Conference on Intelligent Robots and Systems},
  month = nov,
  pages = {6357--6363},
  title = {{Joint Point Cloud and Image Based Localization for Efficient Inspection in Mixed Reality}},
  year = {2018}
}

This paper introduces a method of structure inspection using mixed-reality headsets to reduce the human effort in reporting accurate inspection information such as fault locations in 3D coordinates. Prior to every inspection, the headset needs to be localized. While external pose estimation and fiducial marker based localization would require setup, maintenance, and manual calibration; marker-free self-localization can be achieved using the onboard depth sensor and camera. However, due to limited depth sensor range of portable mixed-reality headsets like Microsoft HoloLens, localization based on simple point cloud registration (sPCR) would require extensive mapping of the environment. Also, localization based on camera image would face same issues as stereo ambiguities and hence depends on viewpoint. We thus introduce a novel approach to Joint Point Cloud and Image-based Localization (JPIL) for mixed-reality headsets that uses visual cues and headset orientation to register small, partially overlapped point clouds and save significant manual labor and time in environment mapping. Our empirical results compared to sPCR show average 10 fold reduction of required overlap surface area that could potentially save on average 20 minutes per inspection. JPIL is not only restricted to inspection tasks but also can be essential in enabling intuitive human-robot interaction for spatial mapping and scene understanding in conjunction with other agents like autonomous robotic systems that are increasingly being deployed in outdoor environments for applications like structural inspection.

@article{Garg:2018tc,
  archiveprefix = {arXiv},
  arxivid = {1812.03539},
  author = {Garg, Rohit and Yang, Shichao and Scherer, Sebastian},
  eprint = {1812.03539},
  journal = {arXiv.org},
  month = dec,
  title = {{Monocular and Stereo Cues for Landing Zone Evaluation for Micro UAVs}},
  url = {http://arxiv.org/abs/1812.03539},
  year = {2018}
}

Autonomous and safe landing is important for unmanned aerial vehicles. We present a monocular and stereo image based method for fast and accurate landing zone evaluation for UAVs in various scenarios. Many existing methods rely on Lidar or depth sensor to provide accurate and dense surface reconstruction. We utilize stereo images to evaluate the slope and monocular images to compute homography error. By combining them together, our approach works for both rigid and non-rigid dynamic surfaces. Experiments on many outdoor scenes such as water, grass and roofs, demonstrate the robustness and effectiveness of our approach.

@article{Gupta:2018tg,
  archiveprefix = {arXiv},
  arxivid = {1809.03565},
  author = {Gupta, Ritwik and Kurtz, Zachary T. and Scherer, Sebastian and Smereka, Jonathon M.},
  eprint = {1809.03565},
  journal = {arXiv.org},
  month = sep,
  title = {{Open Problems in Robotic Anomaly Detection}},
  url = {http://arxiv.org/abs/1809.03565},
  year = {2018}
}

Failures in robotics can have disastrous consequences that worsen rapidly over time. This, the ability to rely on robotic systems, depends on our ability to monitor them and intercede when necessary, manually or autonomously. Prior work in this area surveys intrusion detection and security challenges in robotics, but a discussion of the more general anomaly detection problems is lacking. As such, we provide a brief insight-focused discussion and frameworks of thought on some compelling open problems with anomaly detection in robotic systems. Namely, we discuss non-malicious faults, invalid data, intentional anomalous behavior, hierarchical anomaly detection, distribution of computation, and anomaly correction on the fly. We demonstrate the need for additional work in these areas by providing a case study which examines the limitations of implementing a basic anomaly detection (AD) system in the Robot Operating System (ROS) 2 middleware. Showing that if even supporting a basic system is a significant hurdle, the path to more complex and advanced AD systems is even more problematic. We discuss these ROS 2 platform limitations to support solutions in robotic anomaly detection and provide recommendations to address the issues discovered.

@inproceedings{Schopferer:2018wk,
  address = {Dallas, TX},
  author = {Schopferer, Simon and Lorenz, Julian Soren and Keipour, Azarakhsh and Scherer, Sebastian},
  booktitle = {2018 International Conference on Unmanned Aircraft Systems, ICUAS 2018},
  doi = {10.1109/ICUAS.2018.8453391},
  isbn = {9781538613535},
  keywords = {Fixed-Wing UAV,Path Planning,Trochoids,Wind},
  month = jun,
  pages = {503--512},
  title = {{Path Planning for Unmanned Fixed-Wing Aircraft in Uncertain Wind Conditions Using Trochoids}},
  year = {2018}
}

On-board path planning is a key capability for safe autonomous unmanned flight. Recently, it has been shown that using trochoids for turn segments allows for run-time efficient path planning under consideration of the prevailing wind. However, with varying wind conditions and uncertainty in the estimation of wind speed and direction, paths optimized for a reference wind condition may become infeasible to track for the aircraft. In this work, we discuss how to calculate conservative turn rate limits and we present a novel approach to calculate safety distances along trochoidal turn segments in order to account for an unknown wind speed and airspeed component of bounded magnitude. This allows to plan flight paths that are optimized for the currently expected wind condition but are still safe in case the aircraft experiences different wind conditions. We present results from simulation and flight tests which demonstrate the impact of uncertain and varying wind conditions on the tracking performance of paths with circular and trochoidal turn segments. The results show that trochoids can be used to reduce path tracking errors even if the prevailing wind changes significantly. Furthermore, the proposed method to calculate safety distances conservatively over-approximates path deviations in all considered cases. Thus, it can be used to plan safe paths in the presence of uncertain wind conditions without solely relying on conservative performance limits.

@inproceedings{Tian-2018-110482,
  author = {Tian, Xiangrui and Zhen, Weikun and Scherer, Sebastian and Lu, Xiong},
  booktitle = {50th International Symposium on Robotics, ISR 2018},
  isbn = {9781510870314},
  month = jun,
  pages = {143--146},
  publisher = {VDE},
  title = {{Positioning error analysis of least squares method for wireless sensor networks}},
  year = {2018}
}

Wireless sensor networks (WSN) is widely used for indoor positioning and navigation of mobile robots. Least squares method (LSM) is the most common and simple method for position calculation, and various optimization algorithms were designed elaborately for reducing localization error. Unlike other localization papers which focus on designing elaborate localization algorithms, this paper takes a different perspective, focusing on the error propagation problem, addressing questions such as where the localization error comes from and how it propagates. Based on the theory of variance and covariance, a novel simplified error propagation algorithm is proposed to analyse the localization error for triangulation method. This algorithm exactly shows influence of ranging errors and network structure on positioning. Finally, a simulation test in Matlab is carried out to verify the validity of the proposed algorithm, and it is shown that the algorithm significantly simplifies the calulation of the positioning error. The work of this paper can be used for mul-ti-sensor fusion where an accurate error model is required. Besides, error estimation is also useful for error control by optimizing the structure of WSN.

@incollection{Maturana:2018jq,
  author = {Maturana, Daniel and Chou, Po-Wei and Uenoyama, Masashi and Scherer, Sebastian},
  booktitle = {Field and Service Robotics},
  doi = {10.1007/978-3-319-67361-5_22},
  pages = {335--350},
  publisher = {Springer, Cham},
  title = {{Real-Time Semantic Mapping for Autonomous Off-Road Navigation}},
  year = {2018}
}

In this paper we describe a semantic mapping system for autonomous off-road driving with an All-Terrain Vehicle (ATVs). The system’s goal is to provide a richer representation of the environment than a purely geometric map, allowing it to distinguish, e.g., tall grass from obstacles. The system builds a 2.5D grid map encoding both geometric (terrain height) and semantic information (navigation-relevant classes such as trail, grass, etc.). The geometric and semantic information are estimated online and in real-time from LiDAR and image sensor data, respectively. Using this semantic map, motion planners can create semantically aware trajectories. To achieve robust and efficient semantic segmentation, we design a custom Convolutional Neural Network (CNN) and train it with a novel dataset of labelled off-road imagery built for this purpose. We evaluate our semantic segmentation offline, showing comparable performance to the state of the art with slightly lower latency. We also show closed-loop field results with an autonomous ATV driving over challenging off-road terrain by using the semantic map in conjunction with a simple path planner. Our models and labelled dataset will be publicly available at http://dimatura.net/offroad.

@inproceedings{Liu:2017wv,
  author = {Liu, Xiangzeng and Ai, Yunfeng and Scherer, Sebastian},
  booktitle = {Proceedings - International Conference on Image Processing, ICIP},
  doi = {10.1109/ICIP.2017.8296693},
  isbn = {9781509021758},
  issn = {15224880},
  keywords = {Concrete structure,Crack detection,Guided filter,Image enhancement},
  month = sep,
  pages = {2304--2308},
  title = {{Robust image-based crack detection in concrete structure using multi-scale enhancement and visual features}},
  volume = {2017-September},
  year = {2018}
}

Crack detection is an important technique to evaluate the safety and predict the life of a concrete asset. In order to improve the robustness of the crack detection in complex background, a new crack detection framework based on multi-scale enhancement and visual features is developed. Firstly, to deal with the effect of low contrast, a multi-scale enhancement method using guided filter and gradient information is proposed. Then, the adaptive threshold algorithm is used to obtain the binary image. Finally, the combination of morphological processing and visual features are adopted to purify the cracks. The experimental results with different images of real concrete surface demonstrate the high robustness and validity of the developed technique, in which the average TPR can reach 94.22%.

@incollection{Kim:2018ew,
  author = {Kim, Dong-Ki and Maturana, Daniel and Uenoyama, Masashi and Scherer, Sebastian},
  booktitle = {Field and Service Robotics},
  doi = {10.1007/978-3-319-67361-5_17},
  pages = {255--270},
  publisher = {Springer, Cham},
  title = {{Season-Invariant Semantic Segmentation with a Deep Multimodal Network}},
  year = {2018}
}

Semantic scene understanding is a useful capability for autonomous vehicles operating in off-roads. While cameras are the most common sensor used for semantic classification, the performance of methods using camera imagery may suffer when there is significant variation between the train and testing sets caused by illumination, weather, and seasonal variations. On the other hand, 3D information from active sensors such as LiDAR is comparatively invariant to these factors, which motivates us to investigate whether it can be used to improve performance in this scenario. In this paper, we propose a novel multimodal Convolutional Neural Network (CNN) architecture consisting of two streams, 2D and 3D, which are fused by projecting 3D features to image space to achieve a robust pixelwise semantic segmentation. We evaluate our proposed method in a novel off-road terrain classification benchmark, and show a 25% improvement in mean Intersection over Union (IoU) of navigation-related semantic classes, relative to an image-only baseline.

@inproceedings{Chandarana-2018-107674,
  author = {Chandarana, Meghan and Lewis, Michael and Allen, Bonnie Danette and Sycara, Katia and Scherer, Sebastian},
  booktitle = {2018 Aviation Technology, Integration, and Operations Conference},
  doi = {10.2514/6.2018-3846},
  isbn = {9781624105562},
  month = sep,
  title = {{Swarm size planning tool for multi-job type missions}},
  year = {2018}
}

As part of swarm search and service (SSS) missions, swarms are tasked with searching an area while simultaneously servicing jobs as they are encountered. Jobs must be immediately serviced and can be one of multiple types. Each type requires that vehicle(s) break off from the swarm and travel to the job site for a specified amount of time. The number of vehicles needed and the service time for each job type are known. Once a job has been successfully serviced, vehicles return to the swarm and are available for reallocation. When planning SSS missions, human operators are tasked with determining the required number of vehicles needed to handle the expected job demand. The complex relationship between job type parameters makes this choice challenging. This work presents a prediction model used to estimate the swarm size necessary to achieve a given performance. User studies were conducted to determine the usefulness and ease of use of such a prediction model as an aid during mission planning. Results show that using the planning tool leads to 7x less missed area and a 50% cost reduction.

@article{Khosiawan:2018kf,
  author = {Khosiawan, Yohanes and Scherer, Sebastian and Nielsen, Izabela},
  doi = {10.1177/1687814018815235},
  issn = {16878140},
  journal = {Advances in Mechanical Engineering},
  keywords = {Unmanned aerial vehicle,delay-tolerant,metaheuristic,optimization,scheduling},
  month = dec,
  number = {12},
  pages = {168781401881523},
  title = {{Toward delay-tolerant multiple-unmanned aerial vehicle scheduling system using Multi-strategy Coevolution algorithm}},
  volume = {10},
  year = {2018}
}

Autonomous bridge inspection operations using unmanned aerial vehicles take multiple task assignments and constraints into account. To efficiently execute the operations, a schedule is required. Generating a cost optimum schedule of multiple-unmanned aerial vehicle operations is known to be Non-deterministic Polynomial-time (NP)-hard. This study approaches such a problem with heuristic-based algorithms to get a high-quality feasible solution in a short computation time. A constructive heuristic called Retractable Chain Task Assignment algorithm is presented to build an evaluable schedule from a task sequence. The task sequence representation is used during the search to perform seamless operations. Retractable Chain Task Assignment algorithm calculates and incorporates slack time to the schedule according to the properties of the task. The slack time acts as a cushion which makes the schedule delay-tolerant. This algorithm is incorporated with a metaheuristic algorithm called Multi-strategy Coevolution to search the solution space. The proposed algorithm is verified through numerical simulations, which take inputs from real flight test data. The obtained solutions are evaluated based on the makespan, battery consumption, computation time, and the robustness level of the schedules. The performance of Multi-strategy Coevolution is compared to Differential Evolution, Particle Swarm Optimization, and Differential Evolution–Fused Particle Swarm Optimization. The simulation results show that Multi-strategy Coevolution gives better objective values than the other algorithms.

@inproceedings{Zhu:2018bb,
  author = {Zhu, Jianliang and Ai, Yunfeng and Tian, Bin and Cao, Dongpu and Scherer, Sebastian},
  booktitle = {IEEE Intelligent Vehicles Symposium, Proceedings},
  doi = {10.1109/IVS.2018.8500686},
  isbn = {9781538644522},
  month = oct,
  pages = {1679--1685},
  publisher = {IEEE},
  title = {{Visual Place Recognition in Long-term and Large-scale Environment based on CNN Feature}},
  volume = {2018-June},
  year = {2018}
}

With the universal application of camera in intelligent vehicles, visual place recognition has become a major problem in intelligent vehicle localization. The traditional solution is to make visual description of place images using hand-crafted feature for matching places, but this description method is not very good for extreme variability, especially for seasonal transformation. In this paper, we propose a new method based on convolutional neural network (CNN), by putting images into the pre-trained network model to get automatically learned image descriptors, and through some operations of pooling, fusion and binarization to optimize them, then the similarity result of place recognition is presented with the Hamming distance of the place sequence. In the experimental part, we compare our method with some state-of-the-art algorithms, FABMAP, ABLE-M and SeqSLAM, to illustrate its advantages. The experimental results show that our method based on CNN achieves better performance than other methods on the representative public datasets.

@article{Song:2016gi,
  author = {Song, Yu and Nuske, Stephen and Scherer, Sebastian},
  doi = {10.3390/s17010011},
  issn = {14248220},
  journal = {Sensors (Switzerland)},
  keywords = {Absolute and relative state measurements,GPS-denied state estimation,Long-range stereo visual odometry,Multi-sensor fusion,Stochastic cloning EKF},
  month = dec,
  number = {1},
  pages = {11},
  title = {{A multi-sensor fusion MAV state estimation from long-range stereo, IMU, GPS and barometric sensors}},
  volume = {17},
  year = {2017}
}

State estimation is the most critical capability for MAV (Micro-Aerial Vehicle) localization, autonomous obstacle avoidance, robust flight control and 3D environmental mapping. There are three main challenges for MAV state estimation: (1) it can deal with aggressive 6 DOF (Degree Of Freedom) motion; (2) it should be robust to intermittent GPS (Global Positioning System) (even GPS-denied) situations; (3) it should work well both for low- and high-altitude flight. In this paper, we present a state estimation technique by fusing long-range stereo visual odometry, GPS, barometric and IMU (Inertial Measurement Unit) measurements. The new estimation system has two main parts, a stochastic cloning EKF (Extended Kalman Filter) estimator that loosely fuses both absolute state measurements (GPS, barometer) and the relative state measurements (IMU, visual odometry), and is derived and discussed in detail. A long-range stereo visual odometry is proposed for high-altitude MAV odometry calculation by using both multi-view stereo triangulation and a multi-view stereo inverse depth filter. The odometry takes the EKF information (IMU integral) for robust camera pose tracking and image feature matching, and the stereo odometry output serves as the relative measurements for the update of the state estimation. Experimental results on a benchmark dataset and our real flight dataset show the effectiveness of the proposed state estimation system, especially for the aggressive, intermittent GPS and high-altitude MAV flight.

@inproceedings{Dugar:2017vo,
  address = {Singapore, Singapore},
  author = {Dugar, Vishal and Choudhury, Sanjiban and Scherer, Sebastian},
  booktitle = {Proceedings - IEEE International Conference on Robotics and Automation},
  doi = {10.1109/ICRA.2017.7989017},
  isbn = {9781509046331},
  issn = {10504729},
  month = may,
  pages = {109--116},
  title = {{A $\kappa$iTE in the wind: Smooth trajectory optimization in a moving reference frame}},
  year = {2017}
}

A significant challenge for unmanned aerial vehicles capable of flying long distances is planning in a wind field. Although there has been a plethora of work on the individual topics of planning long routes, smooth trajectory optimization and planning in a wind field, it is difficult for these methods to scale to solve the combined problem. In this paper, we address the problem of planning long, dynamically feasible, time-optimal trajectories in the presence of wind (which creates a moving reference frame). We present an algorithm, \kappaITE, that elegantly decouples the joint trajectory optimization problem into individual path optimization in a fixed ground frame and a velocity profile optimization in a moving reference frame. The key idea is to derive a decoupling framework that guarantees feasibility of the final fused trajectory. Our results show that \kappaITE is able to produce high-quality solutions for planning with a helicopter flying at speeds of 50 m/s, handling winds up to 20 m/s and missions over 200 km. We validate our approach with real-world experiments on a full-scale helicopter with a pilot in the loop. Our approach paves the way forward for autonomous systems to exhibit pilot-like behavior when flying missions in winds aloft.

@article{Choudhury:2017tr,
  annote = {Robotics Science and Systems, 2017},
  archiveprefix = {arXiv},
  arxivid = {1705.07834},
  author = {Choudhury, Sanjiban and Kapoor, Ashish and Ranade, Gireeja and Scherer, Sebastian and Dey, Debadeepta},
  doi = {10.15607/rss.2017.xiii.041},
  eprint = {1705.07834},
  isbn = {9780992374730},
  issn = {2330765X},
  journal = {Robotics: Science and Systems},
  month = may,
  title = {{Adaptive information gathering via imitation learning}},
  volume = {13},
  year = {2017}
}

In the adaptive information gathering problem, a policy is required to select an informative sensing location using the history of measurements acquired thus far. While there is an extensive amount of prior work investigating effective practical approximations using variants of Shannon’s entropy, the efficacy of such policies heavily depends on the geometric distribution of objects in the world. On the other hand, the principled approach of employing online POMDP solvers is rendered impractical by the need to explicitly sample online from a posterior distribution of world maps. We present a novel data-driven imitation learning framework to efficiently train information gathering policies. The policy imitates a clairvoyant oracle - an oracle that at train time has full knowledge about the world map and can compute maximally informative sensing locations. We analyze the learnt policy by showing that offline imitation of a clairvoyant oracle is implicitly equivalent to online oracle execution in conjunction with posterior sampling. This observation allows us to obtain powerful near-optimality guarantees for information gathering problems possessing an adaptive sub-modularity property. As demonstrated on a spectrum of 2D and 3D exploration problems, the trained policies enjoy the best of both worlds - they adapt to different world map distributions while being computationally inexpensive to evaluate.

@article{Yang:2017uf,
  annote = {ICRA 2017},
  author = {Yang, Shichao and Scherer, Sebastian},
  doi = {10.1109/ICRA.2017.7989446},
  isbn = {9781509046331},
  issn = {10504729},
  journal = {Proceedings - IEEE International Conference on Robotics and Automation},
  month = mar,
  pages = {3871--3877},
  title = {{Direct monocular odometry using points and lines}},
  year = {2017}
}

Most visual odometry algorithm for a monocular camera focuses on points, either by feature matching, or direct alignment of pixel intensity, while ignoring a common but important geometry entity: edges. In this paper, we propose an odometry algorithm that combines points and edges to benefit from the advantages of both direct and feature based methods. It works better in texture-less environments and is also more robust to lighting changes and fast motion by increasing the convergence basin. We maintain a depth map for the keyframe then in the tracking part, the camera pose is recovered by minimizing both the photometric error and geometric error to the matched edge in a probabilistic framework. In the mapping part, edge is used to speed up and increase stereo matching accuracy. On various public datasets, our algorithm achieves better or comparable performance than state-of-the-art monocular odometry methods. In some challenging texture-less environments, our algorithm reduces the state estimation error over 50%.

@inproceedings{Dubey:2017wo,
  address = {Vancouver},
  author = {Dubey, Geetesh and Arora, Sankalp and Scherer, Sebastian},
  booktitle = {IEEE International Conference on Intelligent Robots and Systems},
  doi = {10.1109/IROS.2017.8202309},
  isbn = {9781538626825},
  issn = {21530866},
  month = sep,
  pages = {1324--1330},
  title = {{DROAN - Disparity-space representation for obstacle AvoidaNce}},
  volume = {2017-September},
  year = {2017}
}

Agile MAVs are required to operate in cluttered, unstructured environments at high speeds and low altitudes for efficient data gathering. Given the payload constraints and long range sensing requirements, cameras are the preferred sensing modality for MAVs. The computation burden of using cameras for obstacle sensing has forced the state of the art methods to construct world representations on a per frame basis, leading to myopic decision making. In this paper we propose a long range perception and planning approach using cameras. By utilizing FPGA hardware for disparity calculation and image space to represent obstacles, our approach and system design allows for construction of long term world representation whilst accounting for highly non-linear noise models in real time. We demonstrate these obstacle avoidance capabilities on a quadrotor flying through dense foliage at speeds of up to 4 m/s for a total of 1.6 hours of autonomous flights. The presented approach enables high speed navigation at low altitudes for MAVs for terrestrial scouting.

@inproceedings{Chou:2017vd,
  address = {Sydney},
  author = {Chou, Po Wei and Maturana, Daniel and Scherer, Sebastian},
  booktitle = {34th International Conference on Machine Learning, ICML 2017},
  isbn = {9781510855144},
  month = aug,
  pages = {1386--1396},
  title = {{Improving stochastic policy gradients in continuous control with deep reinforcement learning using the beta distribution}},
  volume = {2},
  year = {2017}
}

Recently, reinforcement learning with deep neural networks has achieved great success in challenging continuous control problems such as 3D locomotion and robotic manipulation. However, in real-world control problems, the actions one can take are bounded by physical constraints, which introduces a bias when the standard Gaussian distribution is used as the stochastic policy. In this work, we propose to use the Beta distribution as an alternative and analyze the bias and variance of the policy gradients of both policies. We show that the Beta policy is bias-free and provides significantly faster convergence and higher scores over the Gaussian policy when both are used with trust region policy optimization (TRPO) and actor critic with experience replay (ACER), the state-of-the-art on-and off-policy stochastic methods respectively, on Ope-nAI Gym’s and MuJoCo’s continuous control environments.

@article{Bhardwaj:2017tz,
  archiveprefix = {arXiv},
  arxivid = {1707.03034},
  author = {Bhardwaj, Mohak and Choudhury, Sanjiban and Scherer, Sebastian},
  eprint = {1707.03034},
  journal = {CoRL},
  title = {{Learning Heuristic Search via Imitation}},
  url = {http://arxiv.org/abs/1707.03034},
  year = {2017}
}

Robotic motion planning problems are typically solved by constructing a search tree of valid maneuvers from a start to a goal configuration. Limited onboard computation and real-time planning constraints impose a limit on how large this search tree can grow. Heuristics play a crucial role in such situations by guiding the search towards potentially good directions and consequently minimizing search effort. Moreover, it must infer such directions in an efficient manner using only the information uncovered by the search up until that time. However, state of the art methods do not address the problem of computing a heuristic that explicitly minimizes search effort. In this paper, we do so by training a heuristic policy that maps the partial information from the search to decide which node of the search tree to expand. Unfortunately, naively training such policies leads to slow convergence and poor local minima. We present SaIL, an efficient algorithm that trains heuristic policies by imitating "clairvoyant oracles" - oracles that have full information about the world and demonstrate decisions that minimize search effort. We leverage the fact that such oracles can be efficiently computed using dynamic programming and derive performance guarantees for the learnt heuristic. We validate the approach on a spectrum of environments which show that SaIL consistently outperforms state of the art algorithms. Our approach paves the way forward for learning heuristics that demonstrate an anytime nature - finding feasible solutions quickly and incrementally refining it over time.

@inproceedings{Maturana:2017kd,
  author = {Maturana, Daniel and Arora, Sankalp and Scherer, Sebastian},
  booktitle = {IEEE International Conference on Intelligent Robots and Systems},
  doi = {10.1109/IROS.2017.8206585},
  isbn = {9781538626825},
  issn = {21530866},
  pages = {6691--6698},
  publisher = {IEEE},
  title = {{Looking forward: A semantic mapping system for scouting with micro-aerial vehicles}},
  volume = {2017-September},
  year = {2017}
}

The last decade has seen a massive growth in applications for Micro-Aerial Vehicles (MAVs), due in large part to their versatility for data gathering with cameras, LiDAR and various other sensors. Their ability to quickly go from assessing large spaces from a high vantage points to flying in close to capture high-resolution data makes them invaluable for applications where we are interested in a specific target with an a priori unknown location, e.g. survivors in disaster response scenarios, vehicles in surveillance, animals in wildlife monitoring, etc., a task we will refer to scouting. Our ultimate goal is to enable MAVs to perform autonomous scouting. In this paper, we describe a semantic mapping system designed to support this goal. The system maintains a 2.5D map describing its belief about the location of semantic classes of interest, using forward-looking cameras and state estimation. The map is continuously updated on the fly, using only onboard processing. The system couples a deep learning 2D semantic segmentation algorithm with a novel mapping method to project and aggregate the 2D semantic measurements into a global 2.5D grid map. We train and evaluate our segmentation method on a novel dataset of cars labelled in oblique aerial imagery. We also study the performance of the mapping system in isolation. Finally, we show the integrated system performing a fully autonomous car scouting mission in the field.

@article{Choudhury:2017tka,
  author = {Choudhury, Sanjiban and Javdani, Shervin and Srinivasa, Siddhartha and Scherer, Sebastian},
  issn = {10495258},
  journal = {Advances in Neural Information Processing Systems},
  month = jun,
  pages = {4632--4642},
  title = {{Near-optimal edge evaluation in explicit generalized binomial graphs}},
  volume = {2017-December},
  year = {2017}
}

Robotic motion-planning problems, such as a UAV flying fast in a partially-known environment or a robot arm moving around cluttered objects, require finding collision-free paths quickly. Typically, this is solved by constructing a graph, where vertices represent robot configurations and edges represent potentially valid movements of the robot between these configurations. The main computational bottlenecks are expensive edge evaluations to check for collisions. State of the art planning methods do not reason about the optimal sequence of edges to evaluate in order to find a collision free path quickly. In this paper, we do so by drawing a novel equivalence between motion planning and the Bayesian active learning paradigm of decision region determination (DRD). Unfortunately, a straight application of existing methods requires computation exponential in the number of edges in a graph. We present BISECT, an efficient and near-optimal algorithm to solve the DRD problem when edges are independent Bernoulli random variables. By leveraging this property, we are able to significantly reduce computational complexity from exponential to linear in the number of edges. We show that BISECT outperforms several state of the art algorithms on a spectrum of planning problems for mobile robots, manipulators, and real flight data collected from a full scale helicopter.

@article{Yang:2017we,
  archiveprefix = {arXiv},
  arxivid = {1704.08759},
  author = {Yang, Shichao and Konam, Sandeep and Ma, Chen and Rosenthal, Stephanie and Veloso, Manuela and Scherer, Sebastian},
  eprint = {1704.08759},
  journal = {arXiv.org},
  month = apr,
  title = {{Obstacle Avoidance through Deep Networks based Intermediate Perception}},
  url = {http://arxiv.org/abs/1704.08759},
  year = {2017}
}

Obstacle avoidance from monocular images is a challenging problem for robots. Though multi-view structure-from-motion could build 3D maps, it is not robust in textureless environments. Some learning based methods exploit human demonstration to predict a steering command directly from a single image. However, this method is usually biased towards certain tasks or demonstration scenarios and also biased by human understanding. In this paper, we propose a new method to predict a trajectory from images. We train our system on more diverse NYUv2 dataset. The ground truth trajectory is computed from the designed cost functions automatically. The Convolutional Neural Network perception is divided into two stages: first, predict depth map and surface normal from RGB images, which are two important geometric properties related to 3D obstacle representation. Second, predict the trajectory from the depth and normal. Results show that our intermediate perception increases the accuracy by 20% than the direct prediction. Our model generalizes well to other public indoor datasets and is also demonstrated for robot flights in simulation and experiments.

@inproceedings{Arora:2017tz,
  address = {Singapore, Singapore},
  author = {Arora, Sankalp and Scherer, Sebastian},
  booktitle = {Proceedings - IEEE International Conference on Robotics and Automation},
  doi = {10.1109/ICRA.2017.7989582},
  isbn = {9781509046331},
  issn = {10504729},
  month = may,
  pages = {4997--5004},
  title = {{Randomized algorithm for informative path planning with budget constraints}},
  year = {2017}
}

Maximizing information gathered within a budget is a relevant problem for information gathering tasks for robots with cost or operating time constraints. This problem is also known as the informative path planning (IPP) problem or correlated orienteering. It can be formalized as that of finding budgeted routes in a graph such that the reward collected by the route is maximized, where the reward at nodes can be dependent. Unfortunately, the problem is NP-Hard and the state of the art methods are too slow to even present an approximate solution online. Here we present Randomized Anytime Orienteering (RAOr) algorithm that provides near optimal solutions while demonstrably converging to an efficient solution in runtimes that allows the solver to be run online. The key idea of our approach is to pose orienteering as a combination of a Constraint Satisfaction Problem and a Traveling Salesman Problem. This formulation allows us to restrict the search space to routes that incur minimum distance to visit a set of selected nodes, and rapidly search this space using random sampling. The paper provides the analysis of asymptotic near-optimality, convergence rates for RAOr algorithms, and present strategies to improve anytime performance of the algorithm. Our experimental results suggest an improvement by an order of magnitude over the state of the art methods in relevant simulation and in real world scenarios.

@article{Fang:2017ho,
  author = {Fang, Zheng and Yang, Shichao and Jain, Sezal and Dubey, Geetesh and Roth, Stephan and Maeta, Silvio and Nuske, Stephen and Zhang, Yu and Scherer, Sebastian},
  doi = {10.1002/rob.21670},
  issn = {15564967},
  journal = {Journal of Field Robotics},
  month = jan,
  number = {1},
  pages = {25--52},
  title = {{Robust Autonomous Flight in Constrained and Visually Degraded Shipboard Environments}},
  volume = {34},
  year = {2017}
}

This paper addresses the problem of autonomous navigation of a micro aerial vehicle (MAV) for inspection and damage assessment inside a constrained shipboard environment, which might be perilous or inaccessible for humans, especially in emergency scenarios. The environment is GPS-denied and visually degraded, containing narrow passageways, doorways, and small objects protruding from the wall. This causes existing two-dimensional LIDAR, vision, or mechanical bumper-based autonomous navigation solutions to fail. To realize autonomous navigation in such challenging environments, we first propose a robust state estimation method that fuses estimates from a real-time odometry estimation algorithm and a particle filtering localization algorithm with other sensor information in a two-layer fusion framework. Then, an online motion-planning algorithm that combines trajectory optimization with a receding horizon control framework is proposed for fast obstacle avoidance. All the computations are done in real time on the onboard computer. We validate the system by running experiments under different environmental conditions in both laboratory and practical shipboard environments. The field experiment results of over 10 runs show that our vehicle can robustly navigate 20-m-long and only 1-m-wide corridors and go through a very narrow doorway (66-cm width, only 4-cm clearance on each side) autonomously even when it is completely dark or full of light smoke. These experiments show that despite the challenges associated with flying robustly in challenging shipboard environments, it is possible to use a MAV to autonomously fly into a confined shipboard environment to rapidly gather situational information to guide firefighting and rescue efforts.

@inproceedings{Zhen:2017uf,
  address = {Singapore, Singapore},
  author = {Zhen, Weikun and Zeng, Sam and Soberer, Sebastian},
  booktitle = {Proceedings - IEEE International Conference on Robotics and Automation},
  doi = {10.1109/ICRA.2017.7989739},
  isbn = {9781509046331},
  issn = {10504729},
  month = may,
  pages = {6240--6245},
  title = {{Robust localization and localizability estimation with a rotating laser scanner}},
  year = {2017}
}

This paper presents a robust localization approach that fuses measurements from inertial measurement unit (IMU) and a rotating laser scanner. An Error State Kalman Filter (ESKF) is used for sensor fusion and is combined with a Gaussian Particle Filter (GPF) for measurements update. We experimentally demonstrated the robustness of this implementation in various challenging situations such as kidnapped robot situation, laser range reduction and various environment scales and characteristics. Additionally, we propose a new method to evaluate localizability of a given 3D map and show that the computed localizability can precisely predict localization errors, thus helps to find safe routes during flight.

@article{Shah:2017ut,
  annote = {15 pages, 10 figures. Accepted for presentation in Waste Management Symposium 2018},
  archiveprefix = {arXiv},
  arxivid = {1710.01701},
  author = {Shah, Dhruv and Scherer, Sebastian},
  eprint = {1710.01701},
  journal = {arXiv.org},
  month = oct,
  title = {{Robust Localization of an Arbitrary Distribution of Radioactive Sources for Aerial Inspection}},
  url = {http://arxiv.org/abs/1710.01701},
  year = {2017}
}

Radiation source detection has seen various applications in the past decade, ranging from the detection of dirty bombs in public places to scanning critical nuclear facilities for leakage or flaws, and in the autonomous inspection of nuclear sites. Despite the success in detecting single point sources or a small number of spatially separated point sources, most of the existing algorithms fail to localize sources in complex scenarios with a large number of point sources or non-trivial distributions & bulk sources. Even in simpler environments, most existing algorithms are not scalable to larger regions and/or higher dimensional spaces. For effective autonomous inspection, we not only need to estimate the positions of the sources, but also the number, distribution, and intensities of each of them. In this paper, we present a novel algorithm for the robust localization of an arbitrary distribution of radiation sources using multi-layer sequential Monte Carlo methods coupled with suitable clustering algorithms. We achieve near-perfect accuracy, in terms of F1-scores (\textgreater 0.95), while allowing the algorithm to scale, both to large regions in space and to higher dimensional spaces (5 tested).

@inproceedings{ShichaoYang:2017ufa,
  address = {Vancouver},
  archiveprefix = {arXiv},
  arxivid = {1707.07388},
  author = {Yang, Shichao and Huang, Yulan and Scherer, Sebastian},
  booktitle = {IEEE International Conference on Intelligent Robots and Systems},
  doi = {10.1109/IROS.2017.8202212},
  eprint = {1707.07388},
  isbn = {9781538626825},
  issn = {21530866},
  month = sep,
  pages = {590--597},
  title = {{Semantic 3D occupancy mapping through efficient high order CRFs}},
  volume = {2017-September},
  year = {2017}
}

Semantic 3D mapping can be used for many applications such as robot navigation and virtual interaction. In recent years, there has been great progress in semantic segmentation and geometric 3D mapping. However, it is still challenging to combine these two tasks for accurate and large-scale semantic mapping from images. In the paper, we propose an incremental and (near) real-time semantic mapping system. A 3D scrolling occupancy grid map is built to represent the world, which is memory and computationally efficient and bounded for large scale environments. We utilize the CNN segmentation as prior prediction and further optimize 3D grid labels through a novel CRF model. Superpixels are utilized to enforce smoothness and form robust PN high order potential. An efficient mean field inference is developed for the graph optimization. We evaluate our system on the KITTI dataset and improve the segmentation accuracy by 10% over existing systems.

@inproceedings{Dugar:2017un,
  address = {Fort Worth, TX},
  author = {Dugar, Vishal and Choudhury, Sanjiban and Scherer, Sebastian},
  booktitle = {Annual Forum Proceedings - AHS International},
  issn = {15522938},
  month = may,
  pages = {2924--2932},
  title = {{Smooth trajectory optimization in Wind: First results on a full-scale helicopter}},
  year = {2017}
}

A significant challenge for unmanned aerial vehicles is flying long distances in the presence of wind. The presence of wind, which acts like a forcing function on the system dynamics, significantly affects control authority and flight times. While there is a large body of work on the individual topics of planning long missions and path planning in wind fields, these methods do not scale to solve the combined problem under real-time constraints. In this paper, we address the problem of planning long, dynamically feasible, time-optimal trajectories in the presence of wind for a full-scale helicopter. We build on our existing algorithm, kITE, which accounts for wind in a principled and elegant way, and produces dynamically-feasible trajectories that are guaranteed to be safe in near real-time. It uses a novel framework to decouple path optimization in a fixed ground frame from velocity optimization in a moving air frame. We present extensive experimental evaluation of kITE on an autonomous helicopter platform (with a human safety pilot in the loop) with data from over 23 missions in winds up to 20m=s and airspeeds up to 50m=s. Our results not only shows the efficacy of the algorithm and its implementation, but also provide insights into failure cases that we encountered. This paves the way forward for autonomous systems to exhibit pilot-like behavior when flying missions in winds aloft.

@inproceedings{Madaan:2017wl,
  address = {Vancouver},
  author = {Madaan, Ratnesh and Maturana, Daniel and Scherer, Sebastian},
  booktitle = {IEEE International Conference on Intelligent Robots and Systems},
  doi = {10.1109/IROS.2017.8206190},
  isbn = {9781538626825},
  issn = {21530866},
  month = sep,
  pages = {3487--3494},
  title = {{Wire detection using synthetic data and dilated convolutional networks for unmanned aerial vehicles}},
  volume = {2017-September},
  year = {2017}
}

Wire detection is a key capability for safe navigation of autonomous aerial vehicles and is a challenging problem as wires are generally only a few pixels wide, can appear at any orientation and location, and are hard to distinguish from other similar looking lines and edges. We leverage the recent advances in deep learning by treating wire detection as a semantic segmentation task, and investigate the effectiveness of convolutional neural networks for the same. To find an optimal model in terms of detection accuracy and real time performance on a portable GPU, we perform a grid search over a finite space of architectures. Further, to combat the issue of unavailability of a large public dataset with annotations, we render synthetic wires using a ray tracing engine, and overlay them on 67K images from flight videos available on the internet. We use this synthetic dataset for pretraining our models before finetuning on real data, and show that synthetic data alone can lead to pretty accurate detections qualitatively as well. We also verify if providing explicit information about local evidence of wiry-ness in the form of edge and line detection results from a traditional computer vision method, as additional channels to the network input, makes the task easier or not. We evaluate our best models from the grid search on a publicly available dataset and show that they outperform previous work using traditional computer vision and various deep net baselines of FCNs, SegNet and E-Net, on both standard edge detection metrics and inference speed. Our top models run at more than 3Hz on the NVIDIA Jetson TX2 with input resolution of 480×640, with an Average Precision score of 0.73 on our test split of the USF dataset.

@inproceedings{Pereira:2016jd,
  address = {Washington, D.C.},
  author = {Pereira, Guilherme A.S. and Choudhury, Sanjiban and Scherer, Sebastian},
  booktitle = {2016 International Conference on Unmanned Aircraft Systems, ICUAS 2016},
  doi = {10.1109/ICUAS.2016.7502525},
  isbn = {9781467393331},
  pages = {261--266},
  publisher = {IEEE},
  title = {{A framework for optimal repairing of vector field-based motion plans}},
  year = {2016}
}

This paper presents a framework that integrates vector field based motion planning techniques with an optimal path planner. The main motivation for this integration is the solution of UAVs’ motion planning problems that are easily and intuitively solved using vector fields, but are very difficult to be even posed as optimal motion planning problems, mainly due to the lack of clear cost functions. Examples of such problems include the ones where a goal configuration is not defined, such as circulation of curves, loitering and road following. While several vector field methodologies were proposed to solve these tasks, they are susceptible to failures in the presence of previously unmodeled obstacles, including no-fly zones specified during the flight. Our framework uses a vector field as a high level specification of a task and an optimal motion planner (in our case RRT - ) as a local, on-line planner that generates paths that follow the vector field, but also consider the new obstacles encountered by the vehicle during the flight. A series of simulations illustrate and validate the proposed methodology. One of these simulations considers a rotorcraft UAV equipped with a spinning laser patrolling an urban area in the presence of unmodeled obstacles and no-fly zones.

@techreport{Choudhury:2016uz,
  address = {Pittsburgh, PA},
  author = {Choudhury, Sanjiban and Scherer, Sebastian},
  institution = {Carnegie Mellon University},
  number = {April},
  title = {{Constrained CHOMP using Dual Projected Newton Method}},
  year = {2016}
}

@techreport{Chakraborty:2016tx,
  address = {Pittsburgh, PA},
  author = {Chakraborty, Satyaki and Maturana, Daniel and Scherer, Sebastian},
  institution = {Carnegie Mellon University},
  keywords = {Deconvolution nets,Neural networks,—Object detection},
  number = {CMU-RI-TR-16-60},
  title = {{Detecting cars in aerial photographs with a hierarchy of deconvolution nets}},
  year = {2016}
}

—Detecting cars in large aerial photographs can be quite a challenging task, given that cars in such datasets are often barely visible to the naked human eye. Traditional object detection algorithms fail to perform well when it comes to detecting cars under such circumstances. One would rather use context or exploit spatial relationship between different entities in the scene to narrow down the search space. We aim to do so by looking at different resolutions of the image to process context and focus on promising areas. This is done using a hierarchy of deconvolution networks with each level of the hierarchy trying to predict a heatmap of a certain resolution. We show that our architecture is able to model context implicitly and use it for finer prediction and faster search.

@techreport{Pereira:2016vx,
  address = {Pittsburgh, PA},
  author = {Pereira, Guilherme A S and Choudhury, Sanjiban and Scherer, Sebastian},
  institution = {Carnegie Mellon University},
  keywords = {motion planning,navigation,optimal planners,rrt,vector fields},
  number = {CMU-RI-TR-16-17},
  title = {{Kinodynamic Motion Planning on Vector Fields using RRT *}},
  year = {2016}
}

This report presents a methodology to integrate vector field based motion planning tech-niques with optimal, differential constrained trajectory planners. The main motivation for this integration is the solution of robot motion planning problems that are easily and intuitively solved using vector fields, but are very difficult to be even posed as an optimal motion planning problem, mainly due to the lack of a clear cost function. Examples of such problems include the ones where a goal configuration is not defined, such as circulation of curves, loitering, road following, etc. While several vector field methodologies were proposed to solve these tasks, they do not explicitly consider the robot’s differential constraints and are susceptible to failures in the presence of previously unmodeled obstacles. To add together the good characteristics of each approach, our methodology uses a vector field as a high level specification of a task and an optimal motion planner (in our case RRT*) as a local planner that generates trajectories that follow the vector field, but also consider the kinematics and the dynamics of the robot, as well as the new obstacles encountered by the robot in its workspace.

@inproceedings{Tallavajhula:2016bs,
  address = {Stockholm, Sweden},
  author = {Tallavajhula, Abhijeet and Choudhury, Sanjiban and Scherer, Sebastian and Kelly, Alonzo},
  booktitle = {Proceedings - IEEE International Conference on Robotics and Automation},
  doi = {10.1109/ICRA.2016.7487136},
  isbn = {9781467380263},
  issn = {10504729},
  pages = {213--220},
  publisher = {IEEE},
  title = {{List prediction applied to motion planning}},
  volume = {2016-June},
  year = {2016}
}

There is growing interest in applying machine learning to motion planning. Potential applications are predicting an initial seed for trajectory optimization, predicting an effective heuristic for search based planning, and even predicting a planning algorithm for adaptive motion planning systems. In these situations, providing only a single prediction is unsatisfactory. It leads to many scenarios where the prediction suffers a high loss. In this paper, we advocate list prediction. Each predictor in a list focusses on different regions in the space of environments. This overcomes the shortcoming of a single predictor, and improves overall performance. A framework for list prediction, ConseqOpt, already exists. Our contribution is an extensive domain-specific treatment. We provide a rigorous and clear exposition of the procedure for training a list of predictors. We provide experimental results on a spectrum of motion planning applications. Each application contributes to understanding the behavior of list prediction. We observe that the benefit of list prediction over a single prediction is significant, irrespective of the application.

@techreport{Lee:2016tl,
  address = {Pittsburgh, PA},
  author = {Lee, Richard and Sreenath, Koushil and Scherer, Sebastian},
  institution = {Carnegie Mellon University},
  number = {June},
  title = {{Modeling and Control of Coaxial UAV with Swashplate Controlled Lower Propeller}},
  year = {2016}
}

There is a growing interest in the design and control of coaxial vehicles for the purposes of autonomous flight. These vehicles utilize two, contra-rotating propellers for generating thrust and swashplates for generating pitch and roll. In this report, we present a novel coaxial design in which both upper and lower rotors are contained within a ducted fan, the speeds of both rotors are inde-pendently controlled, and the lower rotor’s cyclic pitch is controlled through a swashplate. Based on this design, a simple dynamic model was developed with unique force and moment generation equations. Given this model, we are able to map desired force and moment values to the control inputs capable of producing them. Afterwards, position and attitude control were implemented over this nonlinear dynamic model in simulation, such that the vehicle was able to recover from poor initial conditions and follow desired trajectories. As demonstrated by the examples presented in this report, position control results in simulations with low max percent overshoot and reasonable settling times. These results prove promising for the implementation of position and attitude control on our physical system.

@inproceedings{Pereira:wi,
  address = {Vitoria, Brazil},
  author = {Pereira, Guilherme A S and Choudhury, Sanjiban and Scherer, Sebastian},
  booktitle = {Congresso Brasileiro de Automatica (CBA)},
  keywords = {este artigo apresenta uma,metodologia para integra,motion planning,optimal planning,resumo,robotics,rrt,vector fields},
  month = oct,
  title = {{Nonholonomic Motion Planning in Partially Unknown Environments Using Vector Fields and Optimal Planners}},
  url = {https://pdfs.semanticscholar.org/28ac/1260c50c9812711914a58ce687e00803cd13.pdf},
  year = {2016}
}

— This paper presents a methodology to integrate vector field-based robot motion planning tech-niques with optimal trajectory planners. The main motivation for this integration is the solution of planning problems that are intuitively solved using vector fields, but are very difficult to be even posed as an optimal mo-tion planning problem, mainly due to the lack of a clear cost function. Among such problems are the ones where a goal configuration is not defined, such as circulation of curves and road following. While several vector field based methodologies were proposed to solve these tasks, they do not explicitly consider the robot’s differential constraints and are susceptible to failures in the presence of previously unmodeled obstacles. Our methodology uses a vector field as a high level specification of a task and an optimal motion planner (in our case RRT*) as a local planner that generates trajectories that follow the vector field and also consider the kinematic and dynamic constraints of the robot, as well as the new obstacles encountered in the workspace. To illustrate the approach, we show simulations with a Dubins like vehicle moving in partially unknown planar environments. Keywords— Robotics, motion planning, vector fields, optimal planning, RRT*. Resumo— Este artigo apresenta uma metodologia para integração de técnicas de planejamento de movimento para robôs baseadas em campos vetorias e planejadore otimos. A principal motivação para essa integraçãó e a solução de problemas de planejamento que são intuitivamente solucionados usando campos vetoriais, mas são muito difícies de serem modelados como um problema de planejamentó otimo, principalmente devidò a falta de uma função de custo. Entre esses problemas estão aqueles em que um alvo nãó e definido, como circulação de curvas e seguimento de rodovias. Enquanto várias metodologias baseadas em campos vetoriais foram propostas para solucionar essas tarefas, elas não consideram explicitamente as restrições diferenciais dos robôs e estão sujeita a falhas na presença de obstáculos que não foram previamente modelados. A metodologia proposta nesse artigo usa um campo vetorial como uma especificação de alto nível para uma tarefa e um planejado otimo (nesse caso o RRT*) como um planejador local que gera trajetórias que seguem o campo vetorial e também consideram as restrições cinemáticas e dinâmicas do robô, bem como os novos obstáculos encontrados em seu espaço de trabalho. Para ilustrar a metodologia, o artigo apresenta simulações utilizando um robô com cinemática de Dubins se locovendo em um ambiente planar parcialmente conhecido. Palavras-chave— Robótica, planejamento de movimento, campos vetorias, planejamentó otimo, RRT*.

@inproceedings{Yang:2016hr,
  address = {Daejeon, Korea},
  archiveprefix = {arXiv},
  arxivid = {1703.07334},
  author = {Yang, Shichao and Song, Yu and Kaess, Michael and Scherer, Sebastian},
  booktitle = {IEEE International Conference on Intelligent Robots and Systems},
  doi = {10.1109/IROS.2016.7759204},
  eprint = {1703.07334},
  isbn = {9781509037629},
  issn = {21530866},
  pages = {1222--1229},
  publisher = {IEEE},
  title = {{Pop-up SLAM: Semantic monocular plane SLAM for low-texture environments}},
  volume = {2016-November},
  year = {2016}
}

Existing simultaneous localization and mapping (SLAM) algorithms are not robust in challenging low-texture environments because there are only few salient features. The resulting sparse or semi-dense map also conveys little information for motion planning. Though some work utilize plane or scene layout for dense map regularization, they require decent state estimation from other sources. In this paper, we propose real-time monocular plane SLAM to demonstrate that scene understanding could improve both state estimation and dense mapping especially in low-texture environments. The plane measurements come from a pop-up 3D plane model applied to each single image. We also combine planes with point based SLAM to improve robustness. On a public TUM dataset, our algorithm generates a dense semantic 3D model with pixel depth error of 6.2 cm while existing SLAM algorithms fail. On a 60 m long dataset with loops, our method creates a much better 3D model with state estimation error of 0.67%.

@inproceedings{ShichaoYang:2016bg,
  address = {Stockholm, Sweden},
  author = {Yang, Shichao and Maturana, Daniel and Scherer, Sebastian},
  booktitle = {Proceedings - IEEE International Conference on Robotics and Automation},
  doi = {10.1109/ICRA.2016.7487368},
  isbn = {9781467380263},
  issn = {10504729},
  pages = {2183--2189},
  publisher = {IEEE},
  title = {{Real-time 3D scene layout from a single image using Convolutional Neural Networks}},
  volume = {2016-June},
  year = {2016}
}

We consider the problem of understanding the 3D layout of indoor corridor scenes from a single image in real time. Identifying obstacles such as walls is essential for robot navigation, but also challenging due to the diversity in structure, appearance and illumination of real-world corridor scenes. Many current single-image methods make Manhattan-world assumptions, and break down in environments that do not meet this mold. They also may require complicated hand-designed features for image segmentation or clear boundaries to form certain building models. In addition, most cannot run in real time In this paper, we propose to combine machine learning with geometric modelling to build a simplified 3D model from a single image.We first employ a supervised Convolutional Neural Network (CNN) to provide a dense, but coarse, geometric class labelling of the scene. We then refine this labelling with a fully connected Conditional Random Field (CRF). Finally, we fit line segments along wall-ground boundaries and ?pop up? a 3D model using geometric constraints. We assemble a dataset of 967 labelled corridor images. Our experiments on this dataset and another publicly available dataset show our method outperforms other single image scene understanding methods in pixelwise accuracy while labelling images at over 15 Hz..

@inproceedings{Choudhury:2016ti,
  address = {Stockholm, Sweden},
  author = {Choudhury, Sanjiban and Gammell, Jonathan D. and Barfoot, Timothy D. and Srinivasa, Siddhartha S.D. and Scherer, Sebastian},
  booktitle = {Proceedings - IEEE International Conference on Robotics and Automation},
  doi = {10.1109/ICRA.2016.7487615},
  isbn = {9781467380263},
  issn = {10504729},
  pages = {4207--4214},
  publisher = {IEEE},
  title = {{Regionally accelerated batch informed trees (RABIT∗): A framework to integrate local information into optimal path planning}},
  volume = {2016-June},
  year = {2016}
}

Sampling-based optimal planners, such as RRT∗, almost-surely converge asymptotically to the optimal solution, but have provably slow convergence rates in high dimensions. This is because their commitment to finding the global optimum compels them to prioritize exploration of the entire problem domain even as its size grows exponentially. Optimization techniques, such as CHOMP, have fast convergence on these problems but only to local optima. This is because they are exploitative, prioritizing the immediate improvement of a path even though this may not find the global optimum of nonconvex cost functions.

@inproceedings{Maturana:2015fs,
  address = {Seattle, WA, USA},
  author = {Maturana, Daniel and Scherer, Sebastian},
  booktitle = {Proceedings - IEEE International Conference on Robotics and Automation},
  doi = {10.1109/ICRA.2015.7139679},
  issn = {10504729},
  number = {June},
  pages = {3471--3478},
  publisher = {IEEE},
  title = {{3D Convolutional Neural Networks for landing zone detection from LiDAR}},
  volume = {2015-June},
  year = {2015}
}

We present a system for the detection of small and potentially obscured obstacles in vegetated terrain. The key novelty of this system is the coupling of a volumetric occupancy map with a 3D Convolutional Neural Network (CNN), which to the best of our knowledge has not been previously done. This architecture allows us to train an extremely efficient and highly accurate system for detection tasks from raw occupancy data. We apply this method to the problem of detecting safe landing zones for autonomous helicopters from LiDAR point clouds. Current methods for this problem rely on heuristic rules and use simple geometric features. These heuristics break down in the presence of low vegetation, as they do not distinguish between vegetation that may be landed on and solid objects that should be avoided. We evaluate the system with a combination of real and synthetic range data. We show our system outperforms various benchmarks, including a system integrating various hand-crafted point cloud features from the literature.

@article{Nuske:2015dj,
  author = {Nuske, Stephen and Choudhury, Sanjiban and Jain, Sezal and Chambers, Andrew and Yoder, Luke and Scherer, Sebastian and Chamberlain, Lyle and Cover, Hugh and Singh, Sanjiv},
  doi = {10.1002/rob.21596},
  issn = {15564967},
  journal = {Journal of Field Robotics},
  number = {8},
  pages = {1141--1162},
  title = {{Autonomous exploration and motion planning for an unmanned aerial vehicle navigating rivers}},
  volume = {32},
  year = {2015}
}

Mapping a river’s geometry provides valuable information to help understand the topology and health of an environment and deduce other attributes such as which types of surface vessels could traverse the river. While many rivers can be mapped from satellite imagery, smaller rivers that pass through dense vegetation are occluded. We develop a micro air vehicle (MAV) that operates beneath the tree line, detects and maps the river, and plans paths around three-dimensional (3D) obstacles (such as overhanging tree branches) to navigate rivers purely with onboard sensing, with no GPS and no prior map. We present the two enabling algorithms for exploration and for 3D motion planning. We extract high-level goal-points using a novel exploration algorithm that uses multiple layers of information to maximize the length of the river that is explored during a mission. We also present an efficient modification to the SPARTAN (Sparse Tangential Network) algorithm called SPARTAN-lite, which exploits geodesic properties on smooth manifolds of a tangential surface around obstacles to plan rapidly through free space. Using limited onboard resources, the exploration and planning algorithms together compute trajectories through complex unstructured and unknown terrain, a capability rarely demonstrated by flying vehicles operating over rivers or over ground. We evaluate our approach against commonly employed algorithms and compare guidance decisions made by our system to those made by a human piloting a boat carrying our system over multiple kilometers. We also present fully autonomous flights on riverine environments generating 3D maps over several hundred-meter stretches of tight winding rivers.

@inproceedings{Jain:wt,
  address = {Brisbanne, Australia},
  author = {Jain, Sezal and Nuske, Stephen and Chambers, Andrew and Yoder, Luke and Cover, Hugh and Chamberlain, Lyle and Scherer, Sebastian and Singh, Sanjiv},
  booktitle = {Springer Tracts in Advanced Robotics},
  doi = {10.1007/978-3-319-07488-7_7},
  isbn = {9783319074870},
  issn = {1610742X},
  month = dec,
  pages = {93--106},
  title = {{Autonomous river exploration}},
  volume = {105},
  year = {2015}
}

Mapping a rivers course and width provides valuable information to help understand the ecology, topology and health of a particular environment. Such maps can also be useful to determine whether specific surface vessels can traverse the rivers. While rivers can be mapped from satellite imagery, the presence of vegetation, sometimes so thick that the canopy completely occludes the river, complicates the process of mapping. Here we propose the use of a micro air vehicle flying under the canopy to create accurate maps of the environment.We study and present a systemthat can autonomously explore riverswithout any prior information, and demonstrate an algorithm that can guide the vehicle based upon local sensors mounted on board the flying vehicle that can perceive the river, bank and obstacles. Our field experiments demonstrate what we believe is the first autonomous exploration of rivers by an autonomous vehicle. We show the 3D maps produced by our system over runs of 100-450 meters in length and compare guidance decisions made by our system to those made by a human piloting a boat carrying our system over multiple kilometers.

@inproceedings{Arora:2015vr,
  author = {Arora, S and Dubey, G and Jain, S and Maturana, D and Song, Y and Nuske, S and Scherer, S},
  booktitle = {Workshop on Vision-based Control and Navigation of Small Lightweight UAVs, IROS 2015},
  month = oct,
  title = {{Autonomous Semantic Exploration Using Unmanned Aerial Vehicles}},
  year = {2015}
}

@inproceedings{Althoff:2015ge,
  address = {Seattle, WA, USA},
  author = {Althoff, Daniel and Scherer, Sebastian},
  booktitle = {Proceedings - IEEE International Conference on Robotics and Automation},
  doi = {10.1109/ICRA.2015.7139651},
  issn = {10504729},
  number = {June},
  pages = {3279--3285},
  publisher = {IEEE},
  title = {{Connected invariant sets for high-speed motion planning in partially-known environments}},
  volume = {2015-June},
  year = {2015}
}

Ensuring safety in partially-known environments is a critical problem in robotics since the environment is perceived through sensors and the environment cannot be completely known ahead of time. Prior work has considered the problem of finding positive control invariant sets (PCIS). However, this approach limits the planning horizon of the motion planner since the PCIS must lie completely in the limited known part of the environment. Here we consider the problem of guaranteeing safety by ensuring the existence of at least one PCIS in partially-known environments leading to an extension of the PCIS concept. It is shown, that this novel method is less conservative than the common PCIS approach and robust to unknown small obstacles which might appear in the close vicinity of the robot. An example implementation for loiter circles and power line obstacles is presented. Simulation scenarios are used for validating the proposed concept.

@inproceedings{Arora:2015fo,
  address = {Seattle, WA, USA},
  author = {Arora, Sankalp and Choudhury, Sanjiban and Althoff, Daniel and Scherer, Sebastian},
  booktitle = {Proceedings - IEEE International Conference on Robotics and Automation},
  doi = {10.1109/ICRA.2015.7140102},
  issn = {10504729},
  number = {June},
  pages = {6431--6438},
  publisher = {IEEE},
  title = {{Emergency maneuver library - Ensuring safe navigation in partially known environments}},
  volume = {2015-June},
  year = {2015}
}

Autonomous mobile robots are required to operate in partially known and unstructured environments. It is imperative to guarantee safety of such systems for their successful deployment. Current state of the art does not fully exploit the sensor and dynamic capabilities of a robot. Also, given the non-holonomic systems with non-linear dynamic constraints, it becomes computationally infeasible to find an optimal solution if the full dynamics are to be exploited online. In this paper we present an online algorithm to guarantee the safety of the robot through an emergency maneuver library. The maneuvers in the emergency maneuver library are optimized such that the probability of finding an emergency maneuver that lies in the known obstacle free space is maximized. We prove that the related trajectory set diversity problem is monotonic and sub-modular which enables one to develop an efficient trajectory set generation algorithm with bounded sub-optimality. We generate an off-line computed trajectory set that exploits the full dynamics of the robot and the known obstacle-free region. We test and validate the algorithm on a full-size autonomous helicopter flying up to speeds of 56\textlesssup\textgreaterm/s\textless/sup\textgreater in partially-known environments. We present results from 4 months of flight testing where the helicopter has been avoiding trees, performing autonomous landing, avoiding mountains while being guaranteed safe.

@techreport{Yang_2015_7839,
  address = {Pittsburgh, PA},
  author = {Yang, Shichao and Fang, Zheng and Jain, Sezal and Dubey, Geetesh and Maeta, Silvio and Roth, Stephan and Scherer, Sebastian and Zhang, Yu and Nuske, Stephen},
  institution = {Carnegie Mellon University},
  month = feb,
  number = {CMU-RI-TR-16-17},
  pages = {CMU--RI--TR--15--06},
  title = {{High-precision Autonomous Flight in Constrained Shipboard Environments}},
  url = {https://www.ri.cmu.edu/pub{\_}files/2015/2/shipboard{\_}final{\_}report{\_}20151.pdf},
  year = {2015}
}

This paper addresses the problem of autonomous navigation of a micro aerial vehicle (MAV) inside of a constrained shipboard environment to aid in fire con-trol, which might be perilous or inaccessible for humans. The environment is GPS-denied and visually degraded, containing narrow passageways, doorways and small objects protruding from the wall, which makes existing 2D LIDAR, vision or mechanical bumper-based autonomous navigation solutions fail. To re-alize autonomous navigation in such challenging environments, we first propose a fast and robust state estimation algorithm that fuses estimates from a direct depth odometry method and a Monte Carlo localization algorithm with other sensor in-formation in a two-level fusion framework. Then, an online motion planning al-gorithm that combines trajectory optimization with receding horizon control is proposed for fast obstacle avoidance. All the computations are done in real-time onboard our customized MAV platform. We validate the system by running ex-periments in different environmental conditions. The results of over 10 runs show that our vehicle robustly navigates 20m long corridors only 1m wide and goes through a very narrow doorway (only 4cm clearance on each side) completely autonomously even when it is completely dark or full of light smoke.

@article{Dorneich:2015kj,
  author = {Dorneich, Michael Christian and Letsu-Dake, Emmanuel and Singh, Sanjiv and Scherer, Sebastian and Chamberlain, Lyle and Bergerman, Marcel},
  doi = {10.5898/jhri.4.3.dorneich},
  issn = {2163-0364},
  journal = {Journal of Human-Robot Interaction},
  month = jan,
  number = {3},
  pages = {38},
  title = {{Mixed-Initiative Control of a Roadable Air Vehicle for Non-Pilots}},
  volume = {4},
  year = {2015}
}

This work developed and evaluated a human-machine interface for the control of a roadable air vehicle (RAV), capable of surface driving, vertical takeoff, sustained flight, and landing. Military applications seek to combine the benefits of ground and air vehicles to maximize flexibility of movement but require that the operator have minimal pilot training. This makes the operator vulnerable to automation complexity issues; however, the operator will expect to be able to interact extensively and control the vehicle during flight. A mixed-initiative control approach mitigates these vulnerabilities by integrating the operator into many complex control domains in the way that they often expect—flexibly in charge, aware, but not required to issue every command. Intrinsic safety aspects were evaluated by comparing performance, decision making, precision, and workload for three RAV control paradigms: human-only, fully automated, and mixed-initiative control. The results suggest that the mixed-initiative paradigm leverages the benefits of human and automated control while also avoiding the drawbacks associated with each.

@techreport{Raj__2015_8025,
  address = {Pittsburgh, PA},
  author = {Raj, Aman and Maturana, Daniel and Scherer, Sebastian},
  booktitle = {Cmu},
  institution = {Carnegie Mellon University},
  month = oct,
  number = {September},
  title = {{Multi-Scale Convolutional Architecture for Semantic Segmentation}},
  url = {https://www.ri.cmu.edu/pub{\_}files/2015/10/CMU-RI-TR{\_}AmanRaj{\_}revision2.pdf},
  year = {2015}
}

Advances in 3D sensing technologies have made the availability of RGB and Depth information easier than earlier which can greatly assist in the semantic segmentation of 2D scenes. There are many works in literature that perform semantic segmentation in such scenes, but few relates to the environment that possesses a high degree of clutter in general e.g. indoor scenes. In this paper, we explore the use of depth information along with RGB and deep convolutional network for indoor scene understanding through semantic labeling. Our work exploits the geocentric encoding of a depth image and uses a multi-scale deep convolutional neural network architecture that captures high and low-level features of a scene to generate rich semantic labels. We apply our method on indoor RGBD images from NYUD2 dataset [1] and achieve a competitive performance of 70.45 % accuracy in labeling four object classes compared with some prior approaches. The results show our system is capable of generating a pixel-map directly from an input image where each pixel-value corresponds to a particular class of object.

@inproceedings{Althoff_2015_8013,
  address = {Hamburg, Germany},
  author = {Althoff, Daniel and Althoff, Matthias and Scherer, Sebastian},
  booktitle = {IEEE International Conference on Intelligent Robots and Systems},
  doi = {10.1109/IROS.2015.7353861},
  isbn = {9781479999941},
  issn = {21530866},
  month = sep,
  pages = {3470--3477},
  title = {{Online safety verification of trajectories for unmanned flight with offline computed robust invariant sets}},
  volume = {2015-December},
  year = {2015}
}

We address the problem of verifying motion plans for aerial robots in uncertain and partially-known environments. Thereby, the initial state of the robot is uncertain due to errors from the state estimation and the motion is uncertain due to wind disturbances and control errors caused by sensor noise. Since the environment is perceived at runtime, the verification of partial motion plans must be performed online (i.e. during operation) to ensure safety within the planning horizon and beyond. This is achieved by efficiently generating robust control invariant sets based on so-called loiter circles, where the position of the aerial robot follows a circular pattern. Verification of aerial robots is challenging due to the nonlinearity of their dynamics, the high dimensionality of their state space, and their potentially high velocities. We use novel techniques from reachability analysis to overcome those challenges. In order to ensure that the robot never finds itself in a situation for which no safe maneuver exists, we provide a technique that ensures safety of aerial robots beyond the planning horizon. Our method is applicable to all kinds of robotic systems that follow reference trajectories, such as bipedal robotic walking, robotic manipulators, automated vehicles, and the like. We evaluate our method by simulations of high speed helicopter flights.

@inproceedings{Arora:2015wx,
  address = {Seattle, WA},
  author = {Arora, Sankalp and Scherer, Sebastian},
  booktitle = {Proceedings - IEEE International Conference on Robotics and Automation},
  doi = {10.1109/ICRA.2015.7139680},
  issn = {10504729},
  month = may,
  number = {June},
  pages = {3479--3486},
  title = {{PASP: Policy based approach for sensor planning}},
  volume = {2015-June},
  year = {2015}
}

Capabilities of mobile autonomous systems is often limited by the sensory constraints. Range sensors moving in a fixed pattern are commonly used as sensing modalities on mobile robots. The performance of these sensors can be augmented by actively controlling their configuration for minimizing the expected cost of the mission. The related information gain problem in NP hard. Current methodologies are either computationally too expensive to run online or make simplifying assumptions that fail in complex environments. We present a method to create and learn a policy that maps features calculated online to sensory actions. The policy developed in this work actively controls a nodding lidar to keep the vehicle safe at high velocities and focuses the sensor bandwidth on gaining information relevant for the mission once safety is ensured. It is validated and evaluated on an autonomous full-scale helicopter (Boeing Unmanned Little Bird) equipped with an actively controlled nodding laser. It is able to keep the vehicle safe at its maximum operating velocity, 56 m=s, and reduce the landing zone evaluation time by 500% as compared to passive nodding. The structure of the policy and efficient learning algorithm should generalize to provide a solution for actively controlling a sensor for keeping a mobile robots safe while exploring regions of interest to the robot.

@inproceedings{Fang:2015ed,
  address = {Seattle, WA, USA},
  author = {Fang, Zheng and Scherer, Sebastian},
  booktitle = {Proceedings - IEEE International Conference on Robotics and Automation},
  doi = {10.1109/ICRA.2015.7139931},
  issn = {10504729},
  number = {June},
  pages = {5253--5259},
  publisher = {IEEE},
  title = {{Real-time onboard 6DoF localization of an indoor MAV in degraded visual environments using a RGB-D camera}},
  volume = {2015-June},
  year = {2015}
}

Real-time and reliable localization is a prerequisite for autonomously performing high-level tasks with micro aerial vehicles(MAVs). Nowadays, most existing methods use vision system for 6DoF pose estimation, which can not work in degraded visual environments. This paper presents an onboard 6DoF pose estimation method for an indoor MAV in challenging GPS-denied degraded visual environments by using a RGB-D camera. In our system, depth images are mainly used for odometry estimation and localization. First, a fast and robust relative pose estimation (6DoF Odometry) method is proposed, which uses the range rate constraint equation and photometric error metric to get the frame-to-frame transform. Then, an absolute pose estimation (6DoF Localization) method is proposed to locate the MAV in a given 3D global map by using a particle filter. The whole localization system can run in real-time on an embedded computer with low CPU usage. We demonstrate the effectiveness of our system in extensive real environments on a customized MAV platform. The experimental results show that our localization system can robustly and accurately locate the robot in various practical challenging environments.

@incollection{Ho:2015id,
  address = {Cham},
  author = {Ju, Jaeyong and Yang, Cheoljong and Scherer, Sebastian and Ko, Hanseok},
  booktitle = {Lecture Notes in Computer Science (including subseries Lecture Notes in Artificial Intelligence and Lecture Notes in Bioinformatics)},
  doi = {10.1007/978-3-319-24078-7_16},
  editor = {Ho, Yo-Sung and Sang, Jitao and Ro, Yong Man and Kim, Junmo and Wu, Fei},
  isbn = {9783319240770},
  issn = {16113349},
  keywords = {Activity recognition,Human group activity,Video analytics},
  month = sep,
  pages = {161--169},
  publisher = {Springer},
  title = {{Recognition of human group activity for video analytics}},
  volume = {9315},
  year = {2015}
}

Human activity recognition is an important and challenging task for video content analysis and understanding. Individual activity recognition has been well studied recently. However, recognizing the activities of human group with more than three people having complex interactions is still a formidable challenge. In this paper, a novel human group activity recognition method is proposed to deal with complex situation where there are multiple sub-groups. To characterize the inherent interactions of intra-subgroups and inter-subgroups with the varying number of participants, this paper proposes three types of group-activity descriptor using motion trajectory and appearance information of people. Experimental results on a public human group activity dataset demonstrate effectiveness of the proposed method.

@inproceedings{Choudhury:2015dw,
  address = {Seattle, WA, USA},
  author = {Choudhury, Sanjiban and Scherer, Sebastian},
  booktitle = {Proceedings - IEEE International Conference on Robotics and Automation},
  doi = {10.1109/ICRA.2015.7139247},
  issn = {10504729},
  number = {June},
  pages = {644--649},
  publisher = {IEEE},
  title = {{The Dynamics Projection Filter (DPF) - Real-time nonlinear trajectory optimization using projection operators}},
  volume = {2015-June},
  year = {2015}
}

Robotic navigation applications often require on-line generation of trajectories that respect underactuated non-linear dynamics, while optimizing a cost function that depends only on a low-dimensional workspace (collision avoidance). Approaches to non-linear optimization, such as differential dynamic programming (DDP), suffer from the drawbacks of slow convergence by being limited to stay within the trust-region of the linearized dynamics and having to integrate the dynamics with fine granularity at each iteration. We address the problem of decoupling the workspace optimization from the enforcement of non-linear constraints. In this paper, we introduce the Dynamics Projection Filter, a nonlinear projection operator based approach that first optimizes a workspace trajectory with reduced constraints and then projects (filters) it to a feasible configuration space trajectory that has a bounded sub-optimality guarantee. We show simulation results for various curvature and curvature-derivatives constrained systems, where the dynamics projection filter is able to, on average, produce similar quality solution 50 times faster than DDP. We also show results from flight tests on an autonomous helicopter that solved these problems on-line while avoiding mountains at high speed as well as trees and buildings as it came in to land.

@inproceedings{Choudhury_2015_7887,
  address = {Seattle, WA, USA},
  author = {Choudhury, Sanjiban and Arora, Sankalp and Scherer, Sebastian},
  booktitle = {Proceedings - IEEE International Conference on Robotics and Automation},
  doi = {10.1109/ICRA.2015.7139517},
  issn = {10504729},
  number = {June},
  pages = {2389--2395},
  publisher = {IEEE},
  title = {{The planner ensemble: Motion planning by executing diverse algorithms}},
  volume = {2015-June},
  year = {2015}
}

Autonomous systems that navigate in unknown environments encounter a variety of planning problems. The success of any one particular planning strategy depends on the validity of assumptions it leverages about the structure of the problem, e.g., Is the cost map locally convex? Does the feasible state space have good connectivity? We address the problem of determining suitable motion planning strategies that can work on a diverse set of applications. We have developed a planning system that does this by running competing planners in parallel. In this paper, we present an approach that constructs a planner ensemble - a set of complementary planners that lever-age a diverse set of assumptions. Our approach optimizes the submodular selection criteria with a greedy approach and lazy evaluation. We seed our selection with learnt priors on planner performance, thus allowing us to solve new applications without evaluating every planner on that application. We present results in simulation where the selected ensemble outperforms the best single planner and does almost as well as an off-line planner. We also present results from an autonomous helicopter that has flown missions several kilometers long at speeds of up to 56m/s which involved avoiding unmapped mountains, no-fly zones and landing in cluttered areas with trees and buildings. This work opens the door on the more general problem of adaptive motion planning.

@article{Choudhury_2015_7965,
  author = {Choudhury, Sanjiban and Scherer, Sebastian and Bagnell, J. Andrew},
  doi = {10.15607/RSS.2015.XI.005},
  isbn = {9780992374716},
  issn = {2330765X},
  journal = {Robotics: Science and Systems},
  title = {{Theoretical limits of speed and resolution for kinodynamic planning in a poisson forest}},
  volume = {11},
  year = {2015}
}

The performance of a state lattice motion planning algorithm depends critically on the resolution of the lattice to ensure a balance between solution quality and computation time. There is currently no theoretical basis for selecting the resolution because of its dependence on the robot dynamics and the distribution of obstacles. In this paper, we examine the problem of motion planning on a resolution constrained lattice for a robot with non-linear dynamics operating in an environment with randomly generated disc shaped obstacles sampled from a homogeneous Poisson process. We present a unified framework for computing explicit solutions to two problems - i) the critical planning resolution which guarantees the existence of an infinite collision free trajectory in the search graph ii) the critical speed limit which guarantees infinite collision free motion. In contrast to techniques used by Karaman and Frazzoli [11], we use a novel approach that maps the problem to parameters of directed asymmetric hexagonal lattice bond percolation. Since standard percolation theory offers no results for this lattice, we map the lattice to an infinite absorbing Markov chain and use results pertaining to its survival to obtain bounds on the parameters. As a result, we are able to derive theoretical expressions that relate the non-linear dynamics of a robot, the resolution of the search graph and the density of the Poisson process. We validate the theoretical bounds using Monte-Carlo simulations for single integrator and curvature constrained systems and are able to validate the previous results presented by Karaman and Frazzoli [11] independently using the novel connections introduced in this paper.

@inproceedings{Maturana_2015_8004,
  address = {Hamburg, Germany},
  author = {Maturana, Daniel and Scherer, Sebastian},
  booktitle = {IEEE International Conference on Intelligent Robots and Systems},
  doi = {10.1109/IROS.2015.7353481},
  isbn = {9781479999941},
  issn = {21530866},
  month = sep,
  pages = {922--928},
  title = {{VoxNet: A 3D Convolutional Neural Network for real-time object recognition}},
  volume = {2015-December},
  year = {2015}
}

Robust object recognition is a crucial skill for robots operating autonomously in real world environments. Range sensors such as LiDAR and RGBD cameras are increasingly found in modern robotic systems, providing a rich source of 3D information that can aid in this task. However, many current systems do not fully utilize this information and have trouble efficiently dealing with large amounts of point cloud data. In this paper, we propose VoxNet, an architecture to tackle this problem by integrating a volumetric Occupancy Grid representation with a supervised 3D Convolutional Neural Network (3D CNN). We evaluate our approach on publicly available benchmarks using LiDAR, RGBD, and CAD data. VoxNet achieves accuracy beyond the state of the art while labeling hundreds of instances per second.

@inproceedings{Arora:2014ul,
  address = {Montreal, CAN},
  author = {Arora, Sankalp and Choudhury, Sanjiban and Althoff, Daniel and Scherer, Sebastian},
  booktitle = {Annual Forum Proceedings - AHS International},
  isbn = {9781632666918},
  issn = {15522938},
  month = may,
  pages = {3228--3236},
  title = {{A principled approach to enable safe and high performance maneuvers for autonomous rotorcraft}},
  volume = {4},
  year = {2014}
}

Autonomous rotorcraft are required to operate in cluttered, unknown, and unstructured environments. Guaranteeing the safety of these systems is critical for their successful deployment. Current methodologies for evaluating or ensuring safety either do not guarantee safety or severely limit the performance of rotorcraft. To design a guaranteed safe rotorcraft, we have defined safety for an autonomous rotorcraft flying in unknown environments given sensory and dynamic constraints. We have developed an approach that ensures the vehicle’s safety while pushing the limits of safe operation of the vehicle. Furthermore, the presented safety definition and the presented approach are independent of the vehicle and planning algorithm used on the rotorcraft. In this paper we present a real time algorithm to guarantee the safety of the rotorcraft through a diverse set of emergency maneuvers. We prove that the related trajectory set diversity problem is monotonic and sub-modular which enables us to develop an efficient, bounded sub-optimal trajectory set generation algorithm. We present safety results for the autonomous Unmanned Little Bird Helicopter flying at speeds of up to 56m/s in partially-known environments. Through months of flight testing the helicopter has been avoiding trees, performing autonomous landing, avoiding mountains while being guaranteed safe. We also present simulation results of the helicopter flying in the Grand Canyon, with no prior map of the environment. Copyright \textcopyright 2014 by the American Helicopter Society International, Inc. All rights reserved.

@inproceedings{Fang:2014be,
  address = {Chicago, IL},
  author = {Fang, Zheng and Scherer, Sebastian},
  booktitle = {IEEE International Conference on Intelligent Robots and Systems},
  doi = {10.1109/IROS.2014.6942632},
  isbn = {9781479969340},
  issn = {21530866},
  pages = {680--687},
  publisher = {IEEE},
  title = {{Experimental study of odometry estimation methods using RGB-D cameras}},
  year = {2014}
}

Lightweight RGB-D cameras that can provide rich 2D visual and 3D point cloud information are well suited to the motion estimation of indoor micro aerial vehicles (MAVs). In recent years, several RGB-D visual odometry methods which process data from the sensor in different ways have been proposed. However, it is unclear which methods are preferable for online odometry estimation on a computation-limited, fast moving MAV in practical indoor environments. This paper presents a detailed analysis and comparison of several state-of-the-art real-time odometry estimation methods in a variety of challenging scenarios, with a special emphasis on the trade-off among accuracy, robustness and computation speed. An experimental comparison is conducted using public available benchmark datasets and author-collected datasets including long corridors, illumination changing environments and fast motion scenarios. Experimental results present both quantitative and qualitative differences among these methods and provide some guidelines on choosing the ’right’ algorithm for an indoor MAV according to the quality of the RGB-D data and environment characteristics.

@techreport{Choudhury_2014_7684,
  address = {Pittsburgh, PA},
  author = {Vemula, Anirudh and Choudhury, Sanjiban and Scherer, Sebastian},
  institution = {Carnegie Mellon University},
  month = aug,
  number = {August},
  pages = {11},
  title = {{Learning Motion Planning Assumptions}},
  url = {https://www.ri.cmu.edu/publication{\_}techreports.html?menu{\_}id=381},
  year = {2014}
}

The performance of a motion planning algorithm is intrinsically linked with applications that respect the assumptions being made. However, the mapping of these assumptions to actual environments is not always transparent. For example, a gradient descent algorithm is capable of tackling a complex opti- mization problem if some assurance of absence of bad local minimas can be ensured - however detecting the local minimas beforehand is very challenging. The state of the art technique relies on an expert to analyze the application, deduce assumptions that the planner can leverage and subsequently make key design decisions. In this work, we make an attempt to learn a mapping from environments to specific planning assumptions. This paper presents a diverse ensemble of planners that exploit very different aspects of the planning problem. A classifier is then trained to approximate the mapping from environment to performance difference between a pair of planners. Preliminary results hints at the role played by convexity, whilst also demonstrating the difficulty of the classification task at hand.

@inproceedings{Chambers:2014kr,
  address = {Portland, OR},
  author = {Chambers, Andrew and Scherer, Sebastian and Yoder, Luke and Jain, Sezal and Nuske, Stephen and Singh, Sanjiv},
  booktitle = {Proceedings of the American Control Conference},
  doi = {10.1109/ACC.2014.6859341},
  isbn = {9781479932726},
  issn = {07431619},
  keywords = {Autonomous systems,Filtering,Vision-based control},
  pages = {1892--1899},
  title = {{Robust multi-sensor fusion for micro aerial vehicle navigation in GPS-degraded/denied environments}},
  year = {2014}
}

State estimation for Micro Air Vehicles (MAVs) is challenging because sensing instrumentation carried on-board is severely limited by weight and power constraints. In addition, their use close to and inside structures and vegetation means that GPS signals can be degraded or all together absent. Here we present a navigation system suited for use on MAVs that seamlessly fuses any combination of GPS, visual odometry, inertial measurements, and/or barometric pressure. We focus on robustness against real-world conditions and evaluate performance in challenging field experiments. Results demonstrate that the proposed approach is effective at providing a consistent state estimate even during multiple sensor failures and can be used for mapping, planning, and control. \textcopyright 2014 American Automatic Control Council.

@inproceedings{Choudhury:2014tq,
  address = {Montreal, CAN},
  author = {Choudhury, Sanjiban and Arora, Sankalp and Scherer, Sebastian},
  booktitle = {Annual Forum Proceedings - AHS International},
  isbn = {9781632666918},
  issn = {15522938},
  month = may,
  pages = {2872--2891},
  title = {{The planner ensemble and trajectory executive: A high performance motion planning system with guaranteed safety}},
  volume = {4},
  year = {2014}
}

Autonomous helicopters are required to fly at a wide range of speed close to ground and eventually land in an unprepared cluttered area. Existing planning systems for unmanned rotorcrafts are capable of flying in unmapped environments, however they are restricted to a specific operating regime dictated by the underlying planning algorithm. We address the problem of planning a trajectory that is computed in real time, respects the dynamics of the helicopter, and keeps the vehicle safe in an unmapped environment with a finite horizon sensor. We have developed a planning system that is capable of doing this by running competing planners in parallel. This paper presents a planning architecture that consists of a trajectory executive - A low latency, verifiable component - That selects plans from a planner ensemble and ensures safety by maintaining emergency maneuvers. Here we report results with an autonomous helicopter that flies missions several kilometers long through unmapped terrain at speeds of upto 56 m/s and landing in clutter. In over 6 months of flight testing, the system has avoided unmapped mountains, popup no fly zones, and has come into land while avoiding trees and buildings in a cluttered landing zone. We also present results from simulation where the same system is flown in challenging obstacle regions - In all cases the system always remains safe and accomplishes the mission. As a result, the system showcases the ability to have a high performance in all environments while guaranteeing safety. \textcopyright 2014 by the American Helicopter Society international Inc. All rights reserved.

@techreport{Maturana_2014_7899,
  address = {Pittsburgh, PA},
  author = {Agarwal, Aditya and Maturana, Daniel and Scherer, Sebastian},
  institution = {Carnegie Mellon University},
  month = jul,
  number = {CMU-RI-TR-15-07},
  pages = {CMU--RI--TR--15--07},
  title = {{Visual Odometry in Smoke Occluded Environments}},
  url = {https://www.ri.cmu.edu/pub{\_}files/2014/7/aditya{\_}tr.pdf},
  year = {2014}
}

@inproceedings{Choudhury:2013vz,
  address = {Phoenix, AZ},
  author = {Choudhury, Sanjiban and Scherer, Sebastian and Singh, Sanjiv},
  booktitle = {Annual Forum Proceedings - AHS International},
  issn = {15522938},
  month = may,
  pages = {2236--2249},
  title = {{Autonomous emergency landing of a helicopter: Motion planning with hard time-constraints}},
  volume = {3},
  year = {2013}
}

Engine malfunctions during helicopter flight poses a large risk to pilot and crew. Without a quick and coordinated reaction, such situations lead to a complete loss of control. An autonomous landing system is capable of reacting quickly to regain control, however current emergency landing methods focus only on the offline generation of dynamically feasible trajectories while ignoring the more severe constraints faced while autonomously landing a real helicopter during an unplanned engine failure. We address the problem of autonomously landing a helicopter while considering a realistic context: hard time-constraints, challenging terrain, sensor limitations and availability of pilot contextual knowledge. We designed a planning system that deals with all these factors by being able to compute alternate routes (AR) in a rapid fashion. This paper presents an algorithm, RRT*-AR, building upon the optimal sampling-based algorithm RRT* to generate AR in realtime while maintaining optimality guarantees and examines its performance for simulated failures occurring in mountainous terrain. After over 4500 trials, RRT*-AR outperformed RRT* by providing the human 280% more options 67% faster on average. As a result, it provides a much wider safety margin for unaccounted disturbances, and a more secure environment for a pilot. \textcopyright 2013 by the American Helicopter Society International, Inc. All rights reserved.

@inproceedings{Mori:2013dm,
  address = {Karlsruhe, Germany},
  author = {Mori, Tomoyuki and Scherer, Sebastian},
  booktitle = {Proceedings - IEEE International Conference on Robotics and Automation},
  doi = {10.1109/ICRA.2013.6630807},
  isbn = {9781467356411},
  issn = {10504729},
  month = may,
  pages = {1750--1757},
  publisher = {IEEE},
  title = {{First results in detecting and avoiding frontal obstacles from a monocular camera for micro unmanned aerial vehicles}},
  year = {2013}
}

Obstacle avoidance is desirable for lightweight micro aerial vehicles and is a challenging problem since the payload constraints only permit monocular cameras and obstacles cannot be directly observed. Depth can however be inferred based on various cues in the image. Prior work has examined optical flow, and perspective cues, however these methods cannot handle frontal obstacles well. In this paper we examine the problem of detecting obstacles right in front of the vehicle. We developed a method to detect relative size changes of image patches that is able to detect size changes in the absence of optical flow. The method uses SURF feature matches in combination with template matching to compare relative obstacle sizes with different image spacing. We present results from our algorithm in autonomous flight tests on a small quadrotor. We are able to detect obstacles with a frame-to-frame enlargement of 120% with a high confidence and confirmed our algorithm in 20 successful flight experiments. In future work, we will improve the control algorithms to avoid more complicated obstacle configurations. \textcopyright 2013 IEEE.

@inproceedings{Arora:2013dg,
  address = {Karlsruhe, Germany},
  author = {Arora, Sankalp and Jain, Sezal and Scherer, Sebastian and Nuske, Stephen and Chamberlain, Lyle and Singh, Sanjiv},
  booktitle = {Proceedings - IEEE International Conference on Robotics and Automation},
  doi = {10.1109/ICRA.2013.6630595},
  isbn = {9781467356411},
  issn = {10504729},
  month = may,
  pages = {323--330},
  publisher = {IEEE},
  title = {{Infrastructure-free shipdeck tracking for autonomous landing}},
  year = {2013}
}

Shipdeck landing is one of the most challenging tasks for a rotorcraft. Current autonomous rotorcraft use shipdeck mounted transponders to measure the relative pose of the vehicle to the landing pad. This tracking system is not only expensive but renders an unequipped ship unlandable. We address the challenge of tracking a shipdeck without additional infrastructure on the deck. We present two methods based on video and lidar that are able to track the shipdeck starting at a considerable distance from the ship. This redundant sensor design enables us to have two independent tracking systems. We show the results of the tracking algorithms in three different environments - field testing results on actual helicopter flights, in simulation with a moving shipdeck for lidar based tracking and in laboratory using an occluded, and, moving scaled model of a landing deck for camera based tracking. The complimentary modalities allow shipdeck tracking under varying conditions. \textcopyright 2013 IEEE.

@article{Chamberlain:2013hn,
  author = {Cooper, Emily},
  doi = {10.1109/MSPEC.2013.6607012},
  issn = {00189235},
  journal = {IEEE Spectrum},
  number = {10},
  pages = {28--33},
  title = {{Robocopters to the rescue}},
  volume = {50},
  year = {2013}
}

We’re standing on the edge of the hot Arizona tarmac, radio in hand, holding our breath as the helicopter passes 50 meters overhead. We watch as the precious sensor on its blunt nose scans every detail of the area, the test pilot and engineer looking down with coolly professional curiosity as they wait for the helicopter to decide where to land. They¿re just onboard observers. The helicopter itself is in charge here. \textcopyright 1964-2012 IEEE.

@inproceedings{Choudhury:2013ek,
  address = {Karlsruhe, Germany},
  author = {Choudhury, Sanjiban and Scherer, Sebastian and Singh, Sanjiv},
  booktitle = {Proceedings - IEEE International Conference on Robotics and Automation},
  doi = {10.1109/ICRA.2013.6631133},
  isbn = {9781467356411},
  issn = {10504729},
  month = may,
  pages = {3947--3952},
  publisher = {IEEE},
  title = {{RRT*-AR: Sampling-based alternate routes planning with applications to autonomous emergency landing of a helicopter}},
  year = {2013}
}

Engine malfunctions during helicopter flight poses a large risk to pilot and crew. Without a quick and coordinated reaction, such situations lead to a complete loss of control. An autonomous landing system could react quicker to regain control, however current emergency landing methods only generate dynamically feasible trajectories without considering obstacles. We address the problem of autonomously landing a helicopter while considering a realistic context: multiple potential landing zones, geographical terrain, sensor limitations and pilot contextual knowledge. We designed a planning system to generate alternate routes (AR) that respect these factors till touchdown exploiting the human-in-loop to make a choice. This paper presents an algorithm, RRT*-AR, building upon the optimal sampling-based algorithm RRT* to generate AR in realtime and examines its performance for simulated failures occurring in mountainous terrain, while maintaining optimality guarantees. After over 4500 trials, RRT*-AR outperformed RRT* by providing the human 280% more options 67% faster on average. As a result, it provides a much wider safety margin for unaccounted disturbances, and a more secure environment for a pilot. Using AR, the focus can now shift on delivering safety guarantees and handling uncertainties in these situations. \textcopyright 2013 IEEE.

@inproceedings{Cover:2013bv,
  address = {Karlsruhe, Germany},
  author = {Cover, Hugh and Choudhury, Sanjiban and Scherer, Sebastian and Singh, Sanjiv},
  booktitle = {Proceedings - IEEE International Conference on Robotics and Automation},
  doi = {10.1109/ICRA.2013.6630967},
  isbn = {9781467356411},
  issn = {10504729},
  month = may,
  pages = {2820--2825},
  publisher = {IEEE},
  title = {{Sparse Tangential Network (SPARTAN): Motion planning for micro aerial vehicles}},
  year = {2013}
}

Micro aerial vehicles operating outdoors must be able to maneuver through both dense vegetation and across empty fields. Existing approaches do not exploit the nature of such an environment. We have designed an algorithm which plans rapidly through free space and is efficiently guided around obstacles. In this paper we present SPARTAN (Sparse Tangential Network) as an approach to create a sparsely connected graph across a tangential surface around obstacles. We find that SPARTAN can navigate a vehicle autonomously through an outdoor environment producing plans 172 times faster than the state of the art (RRT*). As a result SPARTAN can reliably deliver safe plans, with low latency, using the limited computational resources of a lightweight aerial vehicle. \textcopyright 2013 IEEE.

@article{Scherer:2012ke,
  author = {Scherer, Sebastian and Chamberlain, Lyle and Singh, Sanjiv},
  doi = {10.1016/j.robot.2012.09.004},
  issn = {09218890},
  journal = {Robotics and Autonomous Systems},
  keywords = {3D perception,Landing zone selection,Lidar,Rotorcraft,UAV},
  month = dec,
  number = {12},
  pages = {1545--1562},
  title = {{Autonomous landing at unprepared sites by a full-scale helicopter}},
  volume = {60},
  year = {2012}
}

Helicopters are valuable since they can land at unprepared sites; however, current unmanned helicopters are unable to select or validate landing zones (LZs) and approach paths. For operation in unknown terrain it is necessary to assess the safety of a LZ. In this paper, we describe a lidar-based perception system that enables a full-scale autonomous helicopter to identify and land in previously unmapped terrain with no human input. We describe the problem, real-time algorithms, perception hardware, and results. Our approach has extended the state of the art in terrain assessment by incorporating not only plane fitting, but by also considering factors such as terrain/skid interaction, rotor and tail clearance, wind direction, clear approach/abort paths, and ground paths. In results from urban and natural environments we were able to successfully classify LZs from point cloud maps. We also present results from 8 successful landing experiments with varying ground clutter and approach directions. The helicopter selected its own landing site, approaches, and then proceeds to land. To our knowledge, these experiments were the first demonstration of a full-scale autonomous helicopter that selected its own landing zones and landed. \textcopyright 2012 Elsevier Ltd. All rights reserved.

@inproceedings{Scherer:jy,
  address = {St. Paul, MN},
  author = {Scherer, Sebastian and Chamberlain, Lyle and Singh, Sanjiv},
  booktitle = {Proceedings - IEEE International Conference on Robotics and Automation},
  doi = {10.1109/ICRA.2012.6225215},
  isbn = {9781467314039},
  issn = {10504729},
  month = may,
  pages = {951--956},
  publisher = {IEEE},
  title = {{First results in autonomous landing and obstacle avoidance by a full-scale helicopter}},
  year = {2012}
}

Currently deployed unmanned rotorcraft rely on carefully preplanned missions and operate from prepared sites and thus avoid the need to perceive and react to the environment. Here we consider the problems of finding suitable but previously unmapped landing sites given general coordinates of the goal and planning collision free trajectories in real time to land at the "optimal" site. This requires accurate mapping, fast landing zone evaluation algorithms, and motion planning. We report here on the sensing, perception and motion planning integrated onto a full-scale helicopter that flies completely autonomously. We show results from 8 experiments for landing site selection and 5 runs at obstacles. These experiments have demonstrated the first autonomous full-scale helicopter that successfully selects its own landing sites and avoids obstacles. \textcopyright 2012 IEEE.

@techreport{Choudhury:2012ua,
  address = {Pittsburgh, PA},
  author = {Choudhury, Sanjiban and Scherer, Sebastian and Singh, Sanjiv},
  institution = {Carnegie Mellon University},
  number = {December},
  title = {{Realtime alternate routes planning: the rrt*-ar algorithm}},
  url = {http://repository.cmu.edu/robotics/918/?utm{\_}source=repository.cmu.edu/robotics/918{\&}utm{\_}medium=PDF{\&}utm{\_}campaign=PDFCoverPages},
  year = {2012}
}

Motion planning in the most general sense is an optimization problem with a single elusive best solution. However attempting to find a single answer isn’t often the most desired approach. On the one hand, the reason is theoretical - planners often get trapped in local minimas because the cost function has many valleys or dynamics are too complex to fully exploit. On the other hand, there are many practical deterrents - unmapped obstacles might require the system to switch quickly to another plan, unmodelled dynamics can make a computed plan infeasible, or the system may have a human-in-loop who has a vote in the decision process. In situations where the current plan is no longer desirable, a new plan has to be planned. The re-planning time induces a reaction latency which might result in mission failure. We advocate the use of alternate routes (AR), a set of spatially different, locally optimal paths, as a powerful tool to address several of the afore-mentioned issues. By enforcing the routes to be spatially separated, appearance of unexpected obstacles has less chance of rendering all trajectories to be infeasible. In such cases, alternate routes act as a set of backup options which can be switched to instantly. This reduces reaction latency allowing the system to operate with a lower risk. This paper presents an algorithm, RRT*-AR, to generate alternate routes in real time by making tradeoffs in exploitation for exploration, precision for speed and leveraging assumptions about the vehicle and environment constraints. In the case of emergency landing of a helicopter, RRT*-AR outperformed RRT* by providing the human 280% more flight paths 67% faster on average. By planning multiple routes to potential landing zones, the planner was able to seamlessly switch to a new landing site without having to replan.

@article{Scherer:2012hk,
  author = {Scherer, Sebastian and Rehder, Joern and Achar, Supreeth and Cover, Hugh and Chambers, Andrew and Nuske, Stephen and Singh, Sanjiv},
  doi = {10.1007/s10514-012-9293-0},
  issn = {09295593},
  journal = {Autonomous Robots},
  keywords = {3D ladar scanning,3D obstacle mapping,Micro aerial vehicles,Self supervised learning,Visual localization},
  number = {1-2},
  pages = {189--214},
  title = {{River mapping from a flying robot: State estimation, river detection, and obstacle mapping}},
  volume = {33},
  year = {2012}
}

Accurately mapping the course and vegetation along a river is challenging, since overhanging trees block GPS at ground level and occlude the shore line when viewed from higher altitudes. We present a multimodal perception system for the active exploration and mapping of a river from a small rotorcraft. We describe three key components that use computer vision, laser scanning, inertial sensing and intermittant GPS to estimate the motion of the rotorcraft, detect the river without a prior map, and create a 3D map of the riverine environment. Our hardware and software approach is cognizant of the need to perform multi-kilometer missions below tree level with size, weight and power constraints. We present experimental results along a 2 km loop of river using a surrogate perception payload. Overall we can build an accurate 3D obstacle map and a 2D map of the river course and width from light onboard sensing. \textcopyright 2012 Springer Science+Business Media, LLC.

@phdthesis{Scherer:2011tw,
  address = {Pittsburgh, PA},
  author = {Scherer, Sebastian},
  booktitle = {ProQuest Dissertations and Theses},
  isbn = {9781124819532},
  keywords = {0538:Aerospace engineering,0771:Robotics,0800:Artificial intelligence,Aerospace engineering,Applied sciences,Artificial intelligence,Low-altitude operation,Motion planning,Obstacle avoidance,Robotics,Rotorcraft,Unmanned aerial vehicles},
  pages = {138},
  school = {The Robotics Institute, Carnegie Mellon University},
  title = {{Low-Altitude Operation of Unmanned Rotorcraft}},
  url = {http://ezproxy.net.ucf.edu/login?url=http://search.proquest.com/docview/884995854?accountid=10003{\%}5Cnhttp://sfx.fcla.edu/ucf?url{\_}ver=Z39.88-2004{\&}rft{\_}val{\_}fmt=info:ofi/fmt:kev:mtx:dissertation{\&}genre=dissertations+{\&}+theses{\&}sid=ProQ:ProQuest+Dissertations+{\&}+The},
  volume = {3468040},
  year = {2011}
}

Currently deployed unmanned rotorcraft rely on preplanned missions or teleoperation and do not actively incorporate information about obstacles, landing sites, wind, position uncertainty, and other aerial vehicles during online motion planning. Prior work has successfully addressed some tasks such as obstacle avoidance at slow speeds, or landing at known to be good locations. However, to enable autonomous missions in cluttered environments, the vehicle has to react quickly to previously unknown obstacles, respond to changing environmental conditions, and find unknown landing sites. We consider the problem of enabling autonomous operation at low-altitude with contributions to four problems. First we address the problem of fast obstacle avoidance for a small aerial vehicle and present results from over a 1000 rims at speeds up to 10 m/s. Fast response is achieved through a reactive algorithm whose response is learned based on observing a pilot. Second, we show an algorithm to update the obstacle cost expansion for path planning quickly and demonstrate it on a micro aerial vehicle, and an autonomous helicopter avoiding obstacles. Next, we examine the mission of finding a place to land near a ground goal. Good landing sites need to he detected and found and the final touch down goal is unknown. To detect the landing sites we convey a model based algorithm for landing sites that incorporates many helicopter relevant constraints such as landing sites, approach, abort, and ground paths in 3D range data. The landing site evaluation algorithm uses a patch-based coarse evaluation for slope and roughness, and a fine evaluation that fits a 3D model of the helicopter and landing gear to calculate a goodness measure. The data are evaluated in real-time to enable the helicopter to decide on a place to land. We show results from urban, vegetated, and desert environments, and demonstrate the first autonomous helicopter that selects its own landing sites. We present a generalized planning framework that enables reaching a goal point, searching for unknown landing sites, and approaching a landing zone. In the framework, sub-objective functions, constraints, and a state machine define the mission and behavior of an UAV. As the vehicle gathers information by moving through the environment, the objective functions account for this new information. The operator in this framework can directly specify his intent as an objective function that defines the mission rather than giving a sequence of pre-specified goal points. This allows the robot to react to new information received and adjust its path accordingly. The objective is used in a combined coarse planning and trajectory optimization algorithm to determine the best patch the robot should take. We show simulated results for several different missions and in particular focus on active landing zone search. We presented several effective approaches for perception and action for low-altitude flight and demonstrated their effectiveness in field experiments on three autonomous aerial vehicles: a 1m quadrocopter, a 36m helicopter, and a hill-size helicopter. These techniques permit rotorcraft to operate where they have their greatest advantage: In unstructured, unknown environments at low-altitude.

@inproceedings{Scherer_2011_6890,
  address = {San Francisco, CA},
  author = {Scherer, Sebastian and Singh, Sanjiv},
  booktitle = {IEEE International Conference on Intelligent Robots and Systems},
  doi = {10.1109/IROS.2011.6048126},
  isbn = {9781612844541},
  month = sep,
  pages = {2207--2214},
  publisher = {IEEE},
  title = {{Multiple-objective motion planning for unmanned aerial vehicles}},
  year = {2011}
}

Here we consider the problem of low-flying rotorcraft that must perform various missions such as navigating to specific goal points while avoiding obstacles, looking for acceptable landing sites or performing continuous surveillance. Not all of such missions can be expressed as safe, goal seeking, partly because in many cases there isn’t an obvious goal. Rather than developing singular solutions to each mission, we seek a generalized formulation that enables us to express a wider range of missions. Here we propose a framework that allows for multiple objectives to be considered simultaneously and discuss corresponding planning algorithms that are capable of running in realtime on autonomous air vehicles. The algorithms create a set of initial hypotheses that are then optimized by a sub-gradient based trajectory algorithm that optimizes the multiple objectives, producing dynamically feasible trajectories. We have demonstrated the feasibility of our approach with changing cost functions based on newly discovered information. We report on results in simulation of a system that is tasked with navigating safely between obstacles while searching for an acceptable landing site. \textcopyright 2011 IEEE.

@techreport{Molero_Fernandez_2011_6866,
  address = {Pittsburgh, PA},
  author = {Molero, Rudolph and Scherer, Sebastian and Chamberlain, Lyle J},
  booktitle = {Cmu-Ri-Tr-10-08},
  institution = {Carnegie Mellon University},
  keywords = {GPS-denied,MAV,UAV,control,navigation},
  month = jun,
  number = {CMU-RI-TR-10-08},
  title = {{Navigation and Control for Micro Aerial Vehicles in GPS-Denied Environments}},
  year = {2011}
}

Micro-air vehicles have been increasingly employed in diverse research projects in both military and civilian applications. That is because their high maneuverability and accurate mobility. Many of them have been successfully used in outdoor areas, while some have been operated indoors. However, very few have dedicated especial attention to the case of high pitch and roll movements while doing scan-line based odometry. In this paper, we present a general approach consisting of algorithms that enable small aerial robots to fly indoors. We solve the overall problem of large movement change in pitch and roll angles by improving the standard scan matching algorithm. We also validate the effectiveness of the upgraded algorithm by a set of experiments that demonstrate the ability of a small quad-rotor to autonomously operate in cluttered indoor scenarios.

@inproceedings{Chambers_2011_6894,
  address = {San Francisco, CA},
  author = {Chambers, Andrew and Achar, Supreeth and Nuske, Stephen and Rehder, J{\"{o}}rn and Kitt, Bernd and Chamberlain, Lyle and Haines, Justin and Scherer, Sebastian and Singh, Sanjiv},
  booktitle = {IEEE International Conference on Intelligent Robots and Systems},
  doi = {10.1109/IROS.2011.6048799},
  isbn = {9781612844541},
  month = sep,
  pages = {227--234},
  publisher = {IEEE},
  title = {{Perception for a river mapping robot}},
  year = {2011}
}

Rivers with heavy vegetation are hard to map from the air. Here we consider the task of mapping their course and the vegetation along the shores with the specific intent of determining river width and canopy height. A complication in such riverine environments is that only intermittent GPS may be available depending on the thickness of the surrounding canopy. We present a multimodal perception system to be used for the active exploration and mapping of a river from a small rotorcraft flying a few meters above the water. We describe three key components that use computer vision, laser scanning, and inertial sensing to follow the river without the use of a prior map, estimate motion of the rotorcraft, ensure collision-free operation, and create a three dimensional representation of the riverine environment. While the ability to fly simplifies the navigation problem, it also introduces an additional set of constraints in terms of size, weight and power. Hence, our solutions are cognizant of the need to perform multi-kilometer missions with a small payload. We present experimental results along a 2km loop of river using a surrogate system. \textcopyright 2011 IEEE.

@inproceedings{Chamberlain:2011uj,
  address = {Virginia Beach},
  author = {Chamberlain, Lyle and Scherer, Sebastian and Singh, Sanjiv},
  booktitle = {Annual Forum Proceedings - AHS International},
  isbn = {9781617828812},
  issn = {15522938},
  month = mar,
  pages = {3210--3219},
  title = {{Self-aware helicopters: Full-scale automated landing and obstacle avoidance in unmapped environments}},
  volume = {4},
  year = {2011}
}

In this paper we present a perception and autonomy package that for the first time allows a full-scale unmanned helicopter (the Boeing Unmanned Little Bird) to automatically fly through unmapped, obstacle-laden terrain, find a landing zone, and perform a safe landing near a casualty, all with no human control or input. The system also demonstrates the ability to avoid obstacles while in low-altitude flight. The perception system consists of a 3D LADAR mapping unit with sufficient range, accuracy, and bandwidth to bring autonomous flight into the realm of full-scale aircraft. Efficient evaluation of this data and fast planning algorithms provide the aircraft with safe flight trajectories in real-time. We show the results of several fully autonomous landing and obstacle avoidance missions. Copyright \textcopyright 2011, American Helicopter Society International, Inc. All rights reserved.

@inproceedings{achar11,
  address = {Shanghai, China},
  author = {Achar, Supreeth and Sankaran, Bharath and Nuske, Stephen and Scherer, Sebastian and Singh, Sanjiv},
  booktitle = {Proceedings - IEEE International Conference on Robotics and Automation},
  doi = {10.1109/ICRA.2011.5980157},
  isbn = {9781612843865},
  issn = {10504729},
  month = may,
  pages = {6227--6232},
  publisher = {IEEE},
  title = {{Self-supervised segmentation of river scenes}},
  year = {2011}
}

Here we consider the problem of automatically segmenting images taken from a boat or low-flying aircraft. Such a capability is important for autonomous river following and mapping. The need for accurate segmentation in a wide variety of riverine environments challenges the state of the art vision-based methods that have been used in more structured environments such as roads and highways. Apart from the lack of structure, the principal difficulty is the large spatial and temporal variations in the appearance of water in the presence of nearby vegetation and with reflections from the sky. We propose a self-supervised method to segment images into ’sky’, ’river’ and ’shore’ (vegetation + structures) regions. Our approach uses assumptions about river scene structure to learn appearance models based on features like color, texture and image location which are used to segment the image. We validated our algorithm by testing on four datasets captured under varying conditions on different rivers. Our self-supervised algorithm had higher accuracy rates than a supervised alternative, often significantly more accurate, and does not need to be retrained to work under different conditions. \textcopyright 2011 IEEE.

@inproceedings{Scherer:2010wi,
  address = {Atlanta},
  author = {Scherer, Sebastian and Chamberlain, Lyle and Singh, Sanjiv},
  booktitle = {AIAA Infotech at Aerospace 2010},
  doi = {10.2514/6.2010-3358},
  isbn = {9781600867439},
  month = apr,
  title = {{Online assessment of landing sites}},
  year = {2010}
}

Assessing a landing zone (LZ) reliably is essential for safe operation of vertical takeoff and landing (VTOL) aerial vehicles that land at unimproved locations. Currently an operator has to rely on visual assessment to make an approach decision; however. visual information from afar is insufficient to judge slope and detect small obstacles. Prior work has modeled LZ quality based on plane fitting, which only partly represents the interaction between vehicle and ground. Our approach consists of a coarse evaluation based on slope and roughness criteria, a fine evaluation for skid contact, and body clearance of a location. We investigated whether the evaluation is correct for using terrain maps collected from a helicopter. This paper defines the problem of evaluation, describes our incremental real-time algorithm, and discusses the efectiveness of our approach. In results from urban and natural environments, we were able to successfully classify LZs from point cloud maps collected on a helicopter. The presented method enables detailed assessment of LZs without an landing approach, thereby improving safety. Still, the method assumes low-noise point cloud data. We intend to increase robustness to outliers while still detecting small obstacles in future work.

@inproceedings{Scherer:2009hu,
  address = {Kobe, Japan},
  author = {Scherer, Sebastian and Ferguson, Dave and Singh, Sanjiv},
  booktitle = {Proceedings - IEEE International Conference on Robotics and Automation},
  doi = {10.1109/ROBOT.2009.5152790},
  isbn = {9781424427895},
  issn = {10504729},
  month = may,
  pages = {2049--2054},
  publisher = {IEEE},
  title = {{Efficient C-space and cost function updates in 3D for unmanned aerial vehicles}},
  year = {2009}
}

When operating in partially-known environments, autonomous vehicles must constantly update their maps and plans based on new sensor information. Much focus has been placed on developing efficient incremental planning algorithms that are able to efficiently replan when the map and associated cost function changes. However, much less attention has been placed on efficiently updating the cost function used by these planners, which can represent a significant portion of the time spent replanning. In this paper, we present the Limited Incremental Distance Transform algorithm, which can be used to efficiently update the cost function used for planning when changes in the environment are observed. Using this algorithm it is possible to plan paths in a completely incremental way starting from a list of changed obstacle classifications. We present results comparing the algorithm to the Euclidean distance transform and a mask-based incremental distance transform algorithm. Computation time is reduced by an order of magnitude for a UAV application. We also provide example results from an autonomous micro aerial vehicle with on-board sensing and computing.\textcopyright 2009 IEEE.

@techreport{Urmson_2007_6906,
  address = {Pittsburgh, PA},
  author = {Patz, Benjamin J. and Papelis, Yiannis and Pillat, Remo and Stein, Gary and Harper, Don},
  booktitle = {Journal of Field Robotics},
  doi = {10.1002/rob.20251},
  issn = {15564959},
  month = apr,
  number = {8},
  pages = {528--566},
  title = {{A practical approach to robotic design for the DARPA Urban Challenge}},
  url = {http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.107.8722{\&}rep=rep1{\&}type=pdf},
  volume = {25},
  year = {2008}
}

This article presents a practical approach to engineering a robot to effectively navigate in an urban environment. Inherent in this approach is the use of relatively simple sensors, actuators, and processors to generate robot vision, intelligence, and planning. Sensor data are fused from multiple low-cost, two-dimensional laser scanners with an innovative rotational mount to provide three-dimensional coverage with image processing using both range and intensity data. Information is combined with Doppler radar returns to yield a world view processed by a context-based reasoning control system to yield tactical mission commands forwarded to traditional proportional-integral-derivative (PID) control loops. As an example of simplicity and robustness, steering control successfully utilized a relatively simple follow-the-carrot guidance approach that has been successfully demonstrated at speeds of 60 mph (97 km/h). The approach yielded a robot that reached the finals of the Urban Challenge and completed approximately 2 h of the event before being forced to withdraw as a result of a global positioning system data failure. \textcopyright 2008 Wiley Periodicals, Inc.

@inproceedings{SchererSinghChamberlain07,
  address = {Rome, Italy},
  author = {Scherer, Sebastian and Singh, Sanjiv and Chamberlain, Lyle and Elgersma, Mike},
  booktitle = {International Journal of Robotics Research},
  doi = {10.1177/0278364908090949},
  issn = {02783649},
  keywords = {Aerial robotics,Learning},
  month = may,
  number = {5},
  pages = {549--574},
  publisher = {IEEE},
  title = {{Flying fast and low among obstacles: Methodology and experiments}},
  volume = {27},
  year = {2008}
}

Safe autonomous flight is essential for widespread acceptance of aircraft that must fly close to the ground. We have developed a method of collision avoidance that can be used in three dimensions in much the same way as autonomous ground vehicles that navigate over unexplored terrain. Safe navigation is accomplished by a combination of online environmental sensing, path planning and collision avoidance. Here we outline our methodology and report results with an autonomous helicopter that operates at low elevations in uncharted environments, some of which are densely populated with obstacles such as buildings, trees and wires. We have recently completed over 700 successful runs in which the helicopter traveled between coarsely specified waypoints separated by hundreds of meters, at speeds of up to 10 m s-1 at elevations of 5-11 m above ground level. The helicopter safely avoids large objects such as buildings and trees but also wires as thin as 6 mm. We believe this represents the first time an air vehicle has traveled this fast so close to obstacles. The collision avoidance method learns to avoid obstacles by observing the performance of a human operator. \textcopyright SAGE Publications 2008.

@article{Scherer:2008kx,
  author = {Scherer, Sebastian and Singh, Sanjiv and Chamberlain, Lyle and Elgersma, Mike},
  doi = {10.1177/0278364908090949},
  issn = {02783649},
  journal = {International Journal of Robotics Research},
  keywords = {Aerial robotics,Learning},
  number = {5},
  pages = {549--574},
  title = {{Flying fast and low among obstacles: Methodology and experiments}},
  volume = {27},
  year = {2008}
}

Safe autonomous flight is essential for widespread acceptance of aircraft that must fly close to the ground. We have developed a method of collision avoidance that can be used in three dimensions in much the same way as autonomous ground vehicles that navigate over unexplored terrain. Safe navigation is accomplished by a combination of online environmental sensing, path planning and collision avoidance. Here we outline our methodology and report results with an autonomous helicopter that operates at low elevations in uncharted environments, some of which are densely populated with obstacles such as buildings, trees and wires. We have recently completed over 700 successful runs in which the helicopter traveled between coarsely specified waypoints separated by hundreds of meters, at speeds of up to 10 m s-1 at elevations of 5-11 m above ground level. The helicopter safely avoids large objects such as buildings and trees but also wires as thin as 6 mm. We believe this represents the first time an air vehicle has traveled this fast so close to obstacles. The collision avoidance method learns to avoid obstacles by observing the performance of a human operator. \textcopyright SAGE Publications 2008.

@article{Hamner:2006tu,
  author = {Hamner, Bradley and Singh, Sanjiv and Scherer, Sebastian},
  doi = {10.1002/rob.20171},
  issn = {15564959},
  journal = {Journal of Field Robotics},
  number = {11-12},
  pages = {1037--1058},
  title = {{Learning obstacle avoidance parameters from operator behavior}},
  volume = {23},
  year = {2006}
}

This paper concerns an outdoor mobile robot that learns to avoid collisions by observing a human driver operate a vehicle equipped with sensors that continuously produce a map of the local environment. We have implemented steering control that models human behavior in trying to avoid obstacles while trying to follow a desired path. Here we present the formulation for this control system and its independent parameters and then show how these parameters can be automatically estimated by observing a human driver. We also present results from operation on an autonomous robot as well as in simulation, and compare the results from our method to another commonly used learning method. We find that the proposed method generalizes well and is capable of learning from a small number of samples. \textcopyright 2007 Wiley Periodicals, Inc.

@inproceedings{Hamner_2006_5535,
  address = {Beijing, China},
  author = {Hamner, Bradley and Scherer, Sebastian and Singh, Sanjiv},
  booktitle = {IEEE International Conference on Intelligent Robots and Systems},
  doi = {10.1109/IROS.2006.281987},
  isbn = {142440259X},
  month = oct,
  pages = {2663--2669},
  publisher = {IEEE},
  title = {{Learning to drive among obstacles}},
  year = {2006}
}

This paper reports on an outdoor mobile robot that learns to avoid collisions by observing a human driver operate a vehicle equipped with sensors that continuously produce a map of the local environment. We have implemented steering control that models human behavior in trying to avoid obstacles while trying to follow a desired path. Here we present the formulation for this control system and its independent parameters, and then show how these parameters can be automatically estimated by observation of a human driver. We present results from experiments with a vehicle (both real and simulated) that avoids obstacles while following a prescribed path at speeds up to 4 m/sec. We compare the proposed method with another method based on Principal Component Analysis, a commonly used learning technique. We find that the proposed method generalizes well and is capable of learning from a small number of examples. \textcopyright 2006 IEEE.

@inproceedings{Hamner:2005wu,
  author = {Hamner, Bradley and Scherer, Sebastian and Singh, Sanjiv},
  booktitle = {IEEE International Conference on Intelligent Robots and Systems},
  doi = {10.1109/IROS.2006.281987},
  isbn = {142440259X},
  pages = {2663--2669},
  title = {{Learning to drive among obstacles}},
  year = {2006}
}

This paper reports on an outdoor mobile robot that learns to avoid collisions by observing a human driver operate a vehicle equipped with sensors that continuously produce a map of the local environment. We have implemented steering control that models human behavior in trying to avoid obstacles while trying to follow a desired path. Here we present the formulation for this control system and its independent parameters, and then show how these parameters can be automatically estimated by observation of a human driver. We present results from experiments with a vehicle (both real and simulated) that avoids obstacles while following a prescribed path at speeds up to 4 m/sec. We compare the proposed method with another method based on Principal Component Analysis, a commonly used learning technique. We find that the proposed method generalizes well and is capable of learning from a small number of examples. \textcopyright 2006 IEEE.

@inproceedings{Scherer:2005tv,
  address = {Munich, Germany},
  author = {Scherer, S. and Lerda, F. and Clarke, E. M.},
  booktitle = {European Space Agency, (Special Publication) ESA SP},
  issn = {03796566},
  keywords = {Control Systems,Java,Model Checking,Software Testing,Verification},
  month = sep,
  number = {603},
  pages = {371--378},
  title = {{Model checking of robotic control systems}},
  year = {2005}
}

Reliable software is important for robotic applications. We propose a new method for the verification of control software based on Java PathFinder, a discrete model checker developed at NASA Ames Research Center. Our extension of Java PathFinder supports modeling of a real-time scheduler and a physical system, defined in terms of differential equations. This approach not only is able to detect programming errors, like null-pointer dereferences, but also enables the verification of control software whose correctness depends on the physical, real-time environment. We applied this method to the control software of a line-following robot. The verified source code, written in Java, can be executed without any modifications on the microcontroller of the actual robot. Performance evaluation and bug finding are demonstrated on this example.