diff --git a/main.cpp b/main.cpp index 263202dc..4ad84998 100644 --- a/main.cpp +++ b/main.cpp @@ -1,7 +1,7 @@ /* This file is part of WARG's computer-vision - Copyright (c) 2015, Waterloo Aerial Robotics Group (WARG) + Copyright (c) 2015-2017, Waterloo Aerial Robotics Group (WARG) All rights reserved. Redistribution and use in source and binary forms, with or without @@ -46,6 +46,7 @@ #include #include "frame.h" #include "target_identifier.h" +#include "target_analyzer.h" #include "imgimport.h" #include "importer.h" #include "decklink_import.h" @@ -81,6 +82,7 @@ int processors; // Processing module classes Importer importer; TargetIdentifier identifier; +TargetAnalyzer * analyzer = NULL; MetadataInput *logReader = new MetadataInput(); double aveFrameTime = 1000; @@ -163,9 +165,16 @@ void worker(Frame* f) { workers++; assert(!f->get_img().empty()); identifier.process_frame(f); - if (intermediate && f->get_objects().size() > 0) { + int poSize = f->get_objects().size(); + if (intermediate && poSize > 0) { intermediate_buffer.push(f); } + + //Analyze the image after it is identified + analyzer = TargetAnalyzer::getInstance(); + for (int i = 0; i < poSize; i++){ + analyzer->analyze_pixelobject(f->get_objects()[i]); + } workers--; auto end = std::chrono::steady_clock::now(); @@ -323,6 +332,23 @@ vector commands = { }), Command("frames.source.update_delay", "Updates the delay for the source at the given index", {"index", "delay"}, [=](State &newState, vector args) { importer.update_delay(stoi(args[0]), stol(args[1])); + }), + Command("objects.serialize", "Serialize objects to file", {"file_name"}, [=](State &newState, vector args) { + std::ofstream ofs (args[0], std::ofstream::out); + + ofs << "[" << endl; + + vector objects = TargetAnalyzer::getInstance()->extract_objects(); + for (Object *o : objects) { + ofs << o->serialize().str() << endl; + o->write_images(outputDir); + } + ofs << "]" << endl; + ofs.close(); + }), + Command("objects.cluster.species", "Assigns species based on colour", {"num_species"}, [=](State &newState, vector args) { + vector objects = TargetAnalyzer::getInstance()->extract_objects(); + // TODO: Cluster by colour and num_species }) }; diff --git a/modules/core/include/frame.h b/modules/core/include/frame.h index 77fec432..d197309e 100644 --- a/modules/core/include/frame.h +++ b/modules/core/include/frame.h @@ -1,44 +1,26 @@ -/* - This file is part of WARG's computer-vision - Copyright (c) 2015, Waterloo Aerial Robotics Group (WARG) - All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are met: - 1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - 3. Usage of this code MUST be explicitly referenced to WARG and this code - cannot be used in any competition against WARG. - 4. Neither the name of the WARG nor the names of its contributors may be used - to endorse or promote products derived from this software without specific - prior written permission. - - THIS SOFTWARE IS PROVIDED BY WARG ''AS IS'' AND ANY - EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - DISCLAIMED. IN NO EVENT SHALL WARG BE LIABLE FOR ANY - DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*/ - -#ifndef FRAME_H_INCLUDED -#define FRAME_H_INCLUDED - /** * @file frame.h + * @author WARG * * @brief Container class for storing photos or frames of video to be * processed and analyzed * + * @section LICENSE + * + * Copyright (c) 2015-2017, Waterloo Aerial Robotics Group (WARG) + * All rights reserved. + * + * This software is licensed under a modified version of the BSD 3 + * clause license + * that should have been included with this software in a file called + * COPYING.txt + * Otherwise it is available at: + * https://raw.githubusercontent.com/UWARG/computer-vision/master/COPYING.txt */ +#ifndef FRAME_H_INCLUDED +#define FRAME_H_INCLUDED + #include #include "metadata.h" #include @@ -99,6 +81,31 @@ class Frame{ void save(std::string dir); /** + * @brief Getter for the pixel distance + * + * @return The distance covered by each pixel of the image in the X and Y + * directions. + */ + cv::Point2d get_pixel_distance(); + + /** + * @brief Setter for the pixel distance + * + * @param The distance covered by each pixel of the image in the X and Y + * directions. + */ + void set_pixel_distance(cv::Point2d); + + /** + * @brief Setter for the pixel distance + * + * @param The distance covered by each pixel of the image in the X direction. + * @param The distance covered by each pixel of the image in the Y + * direction. + */ + void set_pixel_distance(double x, double y); + + /* * @brief returns a new undistorted image using the given camera * Intended for testing purposes, Does not modify the Frame. * @param camera the camera to use for undistortion @@ -133,6 +140,12 @@ class Frame{ */ std::vector objects; + /** + * @brief Distance of each pixel covers in the x and y direction with + * componesation for altitude, camera lens, etc. + */ + cv::Point2d pixel_distance; + /* * @brief Camera used to capture the frame */ diff --git a/modules/core/include/metadata.h b/modules/core/include/metadata.h index d027bdcb..efd1de42 100644 --- a/modules/core/include/metadata.h +++ b/modules/core/include/metadata.h @@ -1,44 +1,25 @@ -/* - This file is part of WARG's computer-vision - - Copyright (c) 2015, Waterloo Aerial Robotics Group (WARG) - All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are met: - 1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - 3. Usage of this code MUST be explicitly referenced to WARG and this code - cannot be used in any competition against WARG. - 4. Neither the name of the WARG nor the names of its contributors may be used - to endorse or promote products derived from this software without specific - prior written permission. - - THIS SOFTWARE IS PROVIDED BY WARG ''AS IS'' AND ANY - EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - DISCLAIMED. IN NO EVENT SHALL WARG BE LIABLE FOR ANY - DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*/ - -#ifndef METADATA_H_INCLUDED -#define METADATA_H_INCLUDED - /** * @file metadata.h + * @author WARG * * @brief Structure for storing plane status information from the time * that an image was taken * + * @section LICENSE + * + * Copyright (c) 2015-2017, Waterloo Aerial Robotics Group (WARG) + * All rights reserved. + * + * This software is licensed under a modified version of the BSD 3 + * clause license + * that should have been included with this software in a file called + * COPYING.txt + * Otherwise it is available at: + * https://raw.githubusercontent.com/UWARG/computer-vision/master/COPYING.txt */ + +#ifndef METADATA_H_INCLUDED +#define METADATA_H_INCLUDED struct Metadata{ /** diff --git a/modules/core/include/object.h b/modules/core/include/object.h index fd07693c..5962956d 100644 --- a/modules/core/include/object.h +++ b/modules/core/include/object.h @@ -2,39 +2,41 @@ * @file object.h * @author WARG * + * * @section LICENSE * - * Copyright (c) 2015, Waterloo Aerial Robotics Group (WARG) + * Copyright (c) 2015-2017, Waterloo Aerial Robotics Group (WARG) * All rights reserved. * - * This software is licensed under a modified version of the BSD 3 clause license - * that should have been included with this software in a file called COPYING.txt + * This software is licensed under a modified version of the BSD 3 + * clause license + * that should have been included with this software in a file called + * COPYING.txt * Otherwise it is available at: * https://raw.githubusercontent.com/UWARG/computer-vision/master/COPYING.txt */ + #ifndef OBJECT_H_INCLUDED #define OBJECT_H_INCLUDED -/** - * @class Object - * - * @brief Container class for storing information about - * identified targets in real-world measurements - * Adding PixelObjects consolidates their information - * into the Object - * - */ - #include #include class PixelObject; +/* + * @class PixelObject + * @brief Container class for storing information about + * identified targets in real-world measurements + * Adding PixelObjects consolidates their information + * into the Object + */ + class Object { public: - Object(std::string type); + Object(); /** * @brief Getter for Object image @@ -55,7 +57,7 @@ class Object { * * @return GPS co-ordinates of the Object */ - cv::Point2f get_centroid(); + cv::Point2d get_centroid(); /** * @brief Getter for area @@ -83,7 +85,7 @@ class Object { * * @return 2D error magnitude of the Object's location in metres */ - cv::Point2f get_error(); + cv::Point2d get_error(); /** * @brief Getter for error angle @@ -96,9 +98,9 @@ class Object { * @brief Adds given PixelObject to Object's storage * and recalculate target information * - * @param o PixelObject to be added + * @param po PixelObject to be added */ - void add_object(Object * o); + void add_pobject(PixelObject * po); /** * @brief Getter for pixel Objects @@ -106,8 +108,13 @@ class Object { * @return Array containing all of the PixelObjects that were used to * create this instance of Object */ - const std::vector & get_objects(); + const std::vector & get_pobjects(); + + std::stringstream serialize(); + + void write_images(std::string dir); private: + int id; /** * @brief Cropped Image of the Object @@ -127,7 +134,7 @@ class Object { /** * @brief GPS co-ordinates of the centre of the Object */ - cv::Point2f centroid; + cv::Point2d centroid; /** * @brief area of the target in square metres @@ -147,7 +154,7 @@ class Object { /** * @brief Calculated location error of the target as a 2D rectangle in metres */ - cv::Point2f error; + cv::Point2d error; /** * @brief Angle of the error as degrees clockwise from North @@ -159,6 +166,22 @@ class Object { * Each PixelObject is a specific instance of this Object */ std::vector pixelObjects; + + /** + * @brief Updates Object parameters based on the PixelObjects contained + * within it. For now this includes calculating the average + * positions/colours/areas with potential error + * + */ + void update(); + + /** + * @brief Recalculates Object parameters based on the PixelObjects contained + * within it. For now this includes calculating the average + * positions/colours/areas with potential error + * + */ + void recalculate(); }; diff --git a/modules/core/include/pixel_object.h b/modules/core/include/pixel_object.h index 5797bd01..fe55c822 100644 --- a/modules/core/include/pixel_object.h +++ b/modules/core/include/pixel_object.h @@ -4,7 +4,7 @@ * * @section LICENSE * - * Copyright (c) 2015, Waterloo Aerial Robotics Group (WARG) + * Copyright (c) 2015-2017, Waterloo Aerial Robotics Group (WARG) * All rights reserved. * * This software is licensed under a modified version of the BSD 3 clause license @@ -65,6 +65,28 @@ class PixelObject{ * @return Average colour of the interiour of the Object's contour */ cv::Scalar get_colour(); + + /** + * @brief Getter for GPS centroid + * + * @return GPS location of the Object in its frame + */ + cv::Point2d get_gps_centroid(); + + /** + * @brief Setter for GPS centroid + * + * @param gps The GPS centroid of the Object in its frame + */ + void set_gps_centroid(cv::Point2d gps); + + /** + * @brief Getter for area + * + * @return Area of the Object in meters + */ + double get_gps_area(); + /** * @brief Getter for error @@ -103,15 +125,29 @@ class PixelObject{ std::vector contour; /** - * @brief Pixel location of the centre of the Object + * @brief Pixel location of the centre of the PixelObject */ cv::Point2d centroid; + + /** + * @brief GPS location of the centre of the PixelObject. This is calculated + * from PixelObject::centroid after targetanalysis. + */ + cv::Point2d gps_centroid; + /** * @brief area of the target in pixels */ double area; + /** + * @brief area of the target in meters. This is computed after + * targetanalysis has been completed. + */ + double gps_area; + + /** * @brief perimeter of the target in pixels */ @@ -141,6 +177,7 @@ class PixelObject{ * @brief Cropped picture of object */ cv::Mat crop; + }; diff --git a/modules/core/include/target.h b/modules/core/include/target.h index 25259824..15e91cdb 100644 --- a/modules/core/include/target.h +++ b/modules/core/include/target.h @@ -4,7 +4,7 @@ * * @section LICENSE * - * Copyright (c) 2015, Waterloo Aerial Robotics Group (WARG) + * Copyright (c) 2015-2016, Waterloo Aerial Robotics Group (WARG) * All rights reserved. * * This software is licensed under a modified version of the BSD 3 clause license @@ -33,7 +33,7 @@ class Object; class Target{ public: - Target(std::string type); + Target(); /** * @brief Getter for Target image @@ -54,7 +54,16 @@ class Target{ * * @return GPS co-ordinates of the Target */ - cv::Point2f get_centroid(); + cv::Point2d get_centroid(); + + /** + * @brief Getter for the pixel distance + * + * @return The distance covered by each pixel of the image in the X and Y + * directions. + */ + cv::Point2d get_pixel_distance(); + /** * @brief Getter for area @@ -106,6 +115,7 @@ class Target{ * create this instance of Target */ const std::vector & get_objects(); + private: /** @@ -126,7 +136,7 @@ class Target{ /** * @brief GPS co-ordinates of the centre of the Target */ - cv::Point2f centroid; + cv::Point2d centroid; /** * @brief area of the target in square metres diff --git a/modules/core/src/frame.cpp b/modules/core/src/frame.cpp index 09a88269..799f8fb9 100644 --- a/modules/core/src/frame.cpp +++ b/modules/core/src/frame.cpp @@ -87,3 +87,15 @@ void Frame::save(std::string dir) { exifData["Exif.Photo.UserComment"] = ss.str(); image->writeMetadata(); } + +cv::Point2d Frame::get_pixel_distance(){ + return Frame::pixel_distance; +} + +void Frame::set_pixel_distance(double x, double y){ + Frame::pixel_distance = cv::Point2d(x,y); +} + +void Frame::set_pixel_distance(cv::Point2d pd){ + Frame::pixel_distance = pd; +} diff --git a/modules/core/src/object.cpp b/modules/core/src/object.cpp index 079a4f9e..727a2f75 100644 --- a/modules/core/src/object.cpp +++ b/modules/core/src/object.cpp @@ -1,32 +1,143 @@ -/* - This file is part of WARG's computer-vision - - Copyright (c) 2015, Waterloo Aerial Robotics Group (WARG) - All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are met: - 1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - 3. Usage of this code MUST be explicitly referenced to WARG and this code - cannot be used in any competition against WARG. - 4. Neither the name of the WARG nor the names of its contributors may be used - to endorse or promote products derived from this software without specific - prior written permission. - - THIS SOFTWARE IS PROVIDED BY WARG ''AS IS'' AND ANY - EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - DISCLAIMED. IN NO EVENT SHALL WARG BE LIABLE FOR ANY - DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*/ +/** + * @file object.cpp + * @author WARG + * + * @section LICENSE + * + * Copyright (c) 2015-2017, Waterloo Aerial Robotics Group (WARG) + * All rights reserved. + * + * This software is licensed under a modified version of the BSD 3 clause license + * that should have been included with this software in a file called COPYING.txt + * Otherwise it is available at: + * https://raw.githubusercontent.com/UWARG/computer-vision/master/COPYING.txt + */ +#include +#include "object.h" +#include "pixel_object.h" +#include +#include +#include + +using namespace std; +static int idCounter = 0; + +Object::Object(): id(idCounter++){ + +} + + +void Object::add_pobject(PixelObject * po){ + pixelObjects.push_back(po); + + //Update the Object parameters + int n = Object::pixelObjects.size(); + if (n > 1){ + double ratio = (double)n/(n+1); + cv::Point2d maxDistance = this->error + this->centroid; + if (sqrt(pow(maxDistance.x,2) + pow(maxDistance.y,2)) < sqrt(pow(po->get_gps_centroid().x,2) + pow(po->get_gps_centroid().y,2))){ + maxDistance = po->get_gps_centroid(); + } + + this->centroid = this->centroid*ratio + po->get_gps_centroid()/((double)n+1); + this->area = this->area*ratio + po->get_gps_area()/((double)n+1); + this->colour = this->colour*ratio + po->get_colour()/((double)n+1); + + + //TODO: Determine image quality, choose best image + this->error = maxDistance - this->centroid; + this->errorAngle = atan2(this->centroid.y - maxDistance.y, this->centroid.x - maxDistance.x); + } + else{ + recalculate(); + } +} + +cv::Point2d Object::get_centroid(){ + return centroid; +} + +cv::Scalar Object::get_colour(){ + return colour; +} + + +void Object::recalculate(){ + int n = Object::pixelObjects.size(); + this->centroid = cv::Point2d(0,0); + cv::Point2d maxDistance(0,0); + cv::Scalar cSum; + for (PixelObject* po : pixelObjects){ + this->centroid += po->get_gps_centroid(); + this->area += po->get_gps_area(); + cSum += po->get_colour(); + + if (sqrt(pow(maxDistance.x,2) + pow(maxDistance.y,2)) < + sqrt(pow(po->get_gps_centroid().x,2) + pow(po->get_gps_centroid().y,2))){ + maxDistance = po->get_gps_centroid(); + } + //TODO: Determine image quality, choose best image + + } + this->centroid /= n; + this->colour = cSum/n; + this->area /= n; + this->error = maxDistance - this->centroid; + this->errorAngle = atan2(this->centroid.y - maxDistance.y, this->centroid.x - maxDistance.x); +} + +const std::vector& Object::get_pobjects(){ + return pixelObjects; +} + +// Serialize a cv::Mat to a stringstream +stringstream serialize_mat(cv::Mat input) { + // We will need to also serialize the width, height, type and size of the matrix + int width = input.cols; + int height = input.rows; + int type = input.type(); + size_t size = input.total() * input.elemSize(); + + // Initialize a stringstream and write the data + stringstream ss; + ss.write((char*)(&width), sizeof(int)); + ss.write((char*)(&height), sizeof(int)); + ss.write((char*)(&type), sizeof(int)); + ss.write((char*)(&size), sizeof(size_t)); + + // Write the whole image data + ss.write((char*)input.data, size); + + return ss; +} + +stringstream Object::serialize() { + stringstream ss; + ss << "{" << endl; + ss << "\"id\":" << id << "," << endl; + ss << std::setprecision(12) << "\"centroid\":{\"x\":" << centroid.x << ",\"y\":" << centroid.y << "}," << endl; + ss << "\"colour\":{\"b\":" << colour[0] << ",\"g\":" << colour[1] << ",\"r\":" << colour[2] << "}," << endl; + ss << "\"images\":["; + /*for (PixelObject *po : pixelObjects) { + stringstream serializedStream = serialize_mat(po->get_cropped_image()); + base64::encoder E; + stringstream encoded; + E.encode(serializedStream, encoded); + ss << encoded.str() << "," << endl; + }*/ + ss << "]" <get_cropped_image()); + } +} + diff --git a/modules/core/src/pixel_object.cpp b/modules/core/src/pixel_object.cpp index bd8a0208..7d207e5d 100644 --- a/modules/core/src/pixel_object.cpp +++ b/modules/core/src/pixel_object.cpp @@ -4,7 +4,7 @@ * * @section LICENSE * - * Copyright (c) 2015, Waterloo Aerial Robotics Group (WARG) + * Copyright (c) 2015-2017, Waterloo Aerial Robotics Group (WARG) * All rights reserved. * * This software is licensed under a modified version of the BSD 3 clause license @@ -43,6 +43,19 @@ cv::Scalar PixelObject::get_colour(){ return colour; } +cv::Point2d PixelObject::get_gps_centroid(){ + return gps_centroid; +} + +void PixelObject::set_gps_centroid(cv::Point2d gps){ + gps_centroid = gps; +} + +double PixelObject::get_gps_area(){ + return gps_area; +} + + cv::Point2d PixelObject::get_error(){ return error; } diff --git a/modules/core/src/target.cpp b/modules/core/src/target.cpp index bc20736b..87609ee7 100644 --- a/modules/core/src/target.cpp +++ b/modules/core/src/target.cpp @@ -1,32 +1,16 @@ -/* - This file is part of WARG's computer-vision - - Copyright (c) 2015, Waterloo Aerial Robotics Group (WARG) - All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are met: - 1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - 3. Usage of this code MUST be explicitly referenced to WARG and this code - cannot be used in any competition against WARG. - 4. Neither the name of the WARG nor the names of its contributors may be used - to endorse or promote products derived from this software without specific - prior written permission. - - THIS SOFTWARE IS PROVIDED BY WARG ''AS IS'' AND ANY - EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - DISCLAIMED. IN NO EVENT SHALL WARG BE LIABLE FOR ANY - DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*/ +/** + * @file target.cpp + * @author WARG + * + * @section LICENSE + * + * Copyright (c) 2015-2016, Waterloo Aerial Robotics Group (WARG) + * All rights reserved. + * + * This software is licensed under a modified version of the BSD 3 clause license + * that should have been included with this software in a file called COPYING.txt + * Otherwise it is available at: + * https://raw.githubusercontent.com/UWARG/computer-vision/master/COPYING.txt + */ diff --git a/modules/imgimport/include/metadata_input.h b/modules/imgimport/include/metadata_input.h index 78b2078b..3f683ab4 100644 --- a/modules/imgimport/include/metadata_input.h +++ b/modules/imgimport/include/metadata_input.h @@ -4,7 +4,7 @@ * * @section LICENSE * - * Copyright (c) 2015-2016, Waterloo Aerial Robotics Group (WARG) + * Copyright (c) 2015-2017, Waterloo Aerial Robotics Group (WARG) * All rights reserved. * * This software is licensed under a modified version of the BSD 3 clause license diff --git a/modules/imgimport/src/metadata_input.cpp b/modules/imgimport/src/metadata_input.cpp index 08df9978..8069212b 100644 --- a/modules/imgimport/src/metadata_input.cpp +++ b/modules/imgimport/src/metadata_input.cpp @@ -4,7 +4,7 @@ * * @section LICENSE * - * Copyright (c) 2015-2016, Waterloo Aerial Robotics Group (WARG) + * Copyright (c) 2015-2017, Waterloo Aerial Robotics Group (WARG) * All rights reserved. * * This software is licensed under a modified version of the BSD 3 clause license diff --git a/modules/imgimport/src/metadata_reader.cpp b/modules/imgimport/src/metadata_reader.cpp index c0faac06..a884d8bb 100644 --- a/modules/imgimport/src/metadata_reader.cpp +++ b/modules/imgimport/src/metadata_reader.cpp @@ -4,7 +4,7 @@ * * @section LICENSE * - * Copyright (c) 2015-2016, Waterloo Aerial Robotics Group (WARG) + * Copyright (c) 2015-2017, Waterloo Aerial Robotics Group (WARG) * All rights reserved. * * This software is licensed under a modified version of the BSD 3 clause license diff --git a/modules/targetanalysis/CMakeLists.txt b/modules/targetanalysis/CMakeLists.txt index 467c785d..d32c4220 100644 --- a/modules/targetanalysis/CMakeLists.txt +++ b/modules/targetanalysis/CMakeLists.txt @@ -1,6 +1,6 @@ -include_directories(include ../core/include) +include_directories(include ../core/include ../targetid/include) ADD_DEFINITIONS("-DBOOST_LOG_DYN_LINK") -add_library(TargetAnalysis src/area_analyzer.cpp src/target_analyzer.cpp src/target_loader.cpp) -target_link_libraries(TargetAnalysis ${Boost_LIBRARIES} Core) +add_library(TargetAnalysis src/area_analyzer.cpp src/target_analyzer.cpp src/target_loader.cpp src/pixel_object_list.cpp) +target_link_libraries(TargetAnalysis ${Boost_LIBRARIES} Core TargetIdentification) target_compile_features(TargetAnalysis PRIVATE) add_subdirectory("test") diff --git a/modules/targetanalysis/include/object_list.h b/modules/targetanalysis/include/object_list.h new file mode 100644 index 00000000..31cf6aab --- /dev/null +++ b/modules/targetanalysis/include/object_list.h @@ -0,0 +1,75 @@ +/* + This file is part of WARG's computer-vision + + Copyright (c) 2015, Waterloo Aerial Robotics Group (WARG) + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + 3. Usage of this code MUST be explicitly referenced to WARG and this code + cannot be used in any competition against WARG. + 4. Neither the name of the WARG nor the names of its contributors may be used + to endorse or promote products derived from this software without specific + prior written permission. + + THIS SOFTWARE IS PROVIDED BY WARG ''AS IS'' AND ANY + EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL WARG BE LIABLE FOR ANY + DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifndef OBJECT_LIST_H_INCLUDED +#define OBJECT_LIST_H_INCLUDED + +/** + * @file object_list.h + * + * @brief Class which describes a structure for storing pixel targets and then + * later finding a matching set of targets. + * + * Module geolocates targets using their pixel locations + * and photo metadata, determines target type and calculates + * possible error. As targets are processed unique targets will + * be identified and the data combined into a single object. + * + * +**/ + +#include "frame.h" +#include "object.h" + +struct oNode{ + Object* o; + struct oNode* next; + +}; + +class ObjectList +{ +private: + oNode* head; + oNode* tail; + int listLength; +public: + //Constructor + ObjectList(); + //Destructor + ~ObjectList(); + + bool addNode(Object* o); + bool getGPSDuplicates(); + bool getVisualDuplicates(); +}; + +#endif // OBJECT_LIST_H_INCLUDED diff --git a/modules/targetanalysis/include/pixel_object_list.h b/modules/targetanalysis/include/pixel_object_list.h new file mode 100644 index 00000000..68babdf7 --- /dev/null +++ b/modules/targetanalysis/include/pixel_object_list.h @@ -0,0 +1,222 @@ +/** + * @file pixel_object_list.h + * @author WARG + * + * @section LICENSE + * + * Copyright (c) 2015-2017, Waterloo Aerial Robotics Group (WARG) + * All rights reserved. + * + * This software is licensed under a modified version of the BSD 3 clause license + * that should have been included with this software in a file called COPYING.txt + * Otherwise it is available at: + * https://raw.githubusercontent.com/UWARG/computer-vision/master/COPYING.txt + */ + + +#ifndef PIXEL_OBJECT_LIST_H_INCLUDED +#define PIXEL_OBJECT_LIST_H_INCLUDED + +#include "frame.h" +#include "object.h" +#include "pixel_object.h" +#include + +//Macros and Definitions +#define RAD2DEG(rad) ((rad)*180.0/M_PI) +#define DEG2RAD(deg) ((deg)*M_PI/180.0) +#define EARTH_RADIUS 6371000 + +/** + * @class PixelObjectList + * + * PixelObjectList describes a structure for storing pixeltargets and grouping + * similar ones together. The duplicate and unique forms can then be later found + * within a unique 'Object' class. The object class contains all pixel objects + * as well as their averaged data, with accompanying error rates. + * + * Module geolocates targets using their pixel locations, visual contour + * and photo metadata, determines target type and calculates + * possible error. As targets are processed unique targets will + * be identified and the data combined into a single object. + * + * This class is a singleton. + * + * @brief Detects unique visual pixel objects and groups them into object based + * on GPS, visual contour and colour. PixelObjectList is a singleton. + * + */ + +class PixelObjectList{ +/* + * MATCH_THRESHOLD is a fuzzy logic variable, which determines the similarity + * that a Pixel Object must have to another one in terms of the key attributes + */ + +private: + /** + * poNode is an instance of a unique object. As part of a linked list, it + * is linked to other nodes that are deemed also unique. + */ + struct poNode{ + Object* o; + struct poNode* next; + + }; + + /** + * comparitor is a structure used in the processing and sorting of nodes for + * decisions regarding merging duplicate objects or not. + */ + struct comparitor{ + double similarity; + poNode* node; + + }; + /** + * head is the end of the linked list. + */ + poNode* head = NULL; + + /** + * tail is the beginning of the linked list. + */ + poNode* tail = NULL; + /** + * The total length of the list is stored in the listLength variable. + */ + int listLength = 0; + + /* + * firstInstance is a self referencing instance for this singleton class. + */ + static PixelObjectList* firstInstance; + + /* + * COMPARE_AREA is a constant (in principle), which at any point can be + * adjusted. It determines the size of the contours which are compared. It + * is used to maintain cv::Mat structures with optimal dimensions. Reducing + * this value is equivalent to reducing the resolution of the contours and + * would improve computation time. + */ + int COMPARE_AREA; + + /* + * Constructor for PixelObjectList + * + * This is a private constructor due to the singleton instance. Use + * getInstance(), to get or make a new PixelObjectList. + */ + PixelObjectList(){COMPARE_AREA = 400;}; + + /* + * addNode(PixelObject* o) adds a new unique node to the end of the PixelObjectList. + * @param o the pixelobject that is converted to an object and added to the + * list. + */ + void addNode(PixelObject* o); + + /* + * compareGPS(PixelObject* po1, Object* o2) compares the GPS position of + * object based on their proximity to one another. + * @param po1 the pixel object which is being judged and compared against + * already processed objects. + * @param o2 the object which has been deemed to have unique parameter in + * comparison to all other previous objects. + * @return the result from 0 to 1, based on the similarity of the PO and O. + */ + double compareGPS(PixelObject* po1, Object* o2); + + /* + * compareColour(PixelObject* po1, Object* o2) compares the average colour value + * of an object based on previous objects and the criteria for what + * potential colour may be within the scope of detection. + * @param po1 the pixel object which is being judged and compared against + * already processed objects. + * @param o2 the object which has been deemed to have a unique colour in + * comparison to all other previous objects. + * @return the result from 0 to 1, based on the similarity of the PO and O. + */ + double compareColour(PixelObject* po1, Object* o2); + + /* + * compareContours(PixelObject* po1, Object* o2) compares the visual outline + * of an object based on the similarity of the shape and overlap of both + * objects. + * @param po1 the pixel object which is being judged and compared against + * already processed objects. + * @param o2 the object which has been deemed to have unique contours in + * comparison to all other previous objects. + * @return the result from 0 to 1, based on the similarity of the PO and O. + */ + double compareContours(PixelObject* po1, Object* o2); + + /* + * Private operator assignment to prevent mismanagement of singleton + * instance. + */ + PixelObjectList& operator=(PixelObjectList*){}; // Private assignment operator + + /* + * Private copy constructor to prevent mismanagement of singleton instance. + */ + PixelObjectList(PixelObjectList const&){}; + + /* + * Destructor for PixelObjectList. + */ + ~PixelObjectList(); +public: + /* + * getInstance() returns the singleton instance of this class. If it is not + * instantiated it is initialized. + * @return the singleton PixelObjectList + */ + static PixelObjectList* getInstance(){ + if (!firstInstance){ + firstInstance = new PixelObjectList; + } + return firstInstance; + } + + /* + * compareNode(PixelObject* po1, Object* o2) compares po1 to the + * pixelobjects within o2, however many they may be. + * @param po1 the pixel object which is being judged and compared against + * already processed objects. + * @param o2 the object which has been deemed to have unique parameter in + * comparison to all other previous objects. + * @return the result from 0 to 1, based on the similarity of the PO and O. + */ + double compareNode(PixelObject* po1, Object* o2); + + /* + * addAndCompare(PixelObject* po) iterates the comparison over all known + * Objects based on the comparison critera. The decision to add a new object + * to the end of the list, or push it into a duplicate is decided here based + * on some fuzzy logic. + * @param po the PixelObject that is being compared and judged. + */ + void addAndCompare(PixelObject* po); + + /* + * getListLength() is a size getter. + * @return provides a size value of the current list + */ + int getListLength(); + + /* + * getObject(int index) is a getter for unique objects based on their + * position in the list. + * @param index 0-based index for getting unique targets from the list + * @return a pointer to the unique object requested. + */ + Object* getObject(int index); + + /* + * getObjects(std::vector* v) returns ALL objects. + * @param v a return parameter to all the objects in the list. + */ + void getObjects(std::vector* v); +}; +#endif // PIXEL_OBJECT_LIST_H_INCLUDED diff --git a/modules/targetanalysis/include/target_analyzer.h b/modules/targetanalysis/include/target_analyzer.h index d25a7ed1..5e2cdae4 100644 --- a/modules/targetanalysis/include/target_analyzer.h +++ b/modules/targetanalysis/include/target_analyzer.h @@ -1,33 +1,18 @@ -/* - This file is part of WARG's computer-vision - - Copyright (c) 2015, Waterloo Aerial Robotics Group (WARG) - All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are met: - 1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - 3. Usage of this code MUST be explicitly referenced to WARG and this code - cannot be used in any competition against WARG. - 4. Neither the name of the WARG nor the names of its contributors may be used - to endorse or promote products derived from this software without specific - prior written permission. - - THIS SOFTWARE IS PROVIDED BY WARG ''AS IS'' AND ANY - EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - DISCLAIMED. IN NO EVENT SHALL WARG BE LIABLE FOR ANY - DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*/ +/** + * @file target_analyzer.h + * @author WARG + * + * @section LICENSE + * + * Copyright (c) 2015-2017, Waterloo Aerial Robotics Group (WARG) + * All rights reserved. + * + * This software is licensed under a modified version of the BSD 3 clause license + * that should have been included with this software in a file called COPYING.txt + * Otherwise it is available at: + * https://raw.githubusercontent.com/UWARG/computer-vision/master/COPYING.txt + */ + #ifndef TARGET_ANALYZER_H_INCLUDED #define TARGET_ANALYZER_H_INCLUDED @@ -48,35 +33,180 @@ **/ #include "frame.h" +#include "pixel_object.h" +#include "object.h" +#include +#include + +//Classes +/** + * @class TargetAnalyzer + * + * TargetAnalyzer is a wrapper module for the PixelObjectList class. It handles + * the entry and exit points for every form of interaction with the target + * analysis module. + * This class is a singleton. + * + * @brief Provides a procedure for analyzing pixel objects, as well as + * extracting data from the results. PixelObjectList is a singleton. + * + */ class TargetAnalyzer { + /* + * analyzer is the static singleton instance of the TargetAnalyzer class + */ static TargetAnalyzer * analyzer; - TargetAnalyzer(); + /* + * Private constructor due to singleton design pattern. + */ + TargetAnalyzer(){}; + + /* + * Settings variables for threshold and threshold biasing in the + * grouping algorithm + * + * VISUAL_THRESHOLD determines how similar two contours need to be inorder + * to be designated a 'duplicate'. + * + * GPS_THRESHOLD determines how similar two GPS coordinates must be inorder + * to be designated a 'duplicate'. A value of 0.1 coorresponds to 10 meter + * accuracy (1/0.1 = 10m) + * + * COLOUR_THRESHOLD determines how similar two pixel object colours must be inorder + * to be designated a 'duplicate'. This is based on a relative RGB scale, + * where White and Black are the polar opposites and would be designated a 0 + * (based on a euclidean RGB distance). + * + * GPS_THRESHOLD_BIAS determines how important GPS similarity is in + * comparison to VISUAL and COLOR parameters. This value is applied AFTER it matches + * the original GPS_THRESHOLD. + * + * VISUAL_THRESHOLD_BIAS determines how important GPS similarity is in + * comparison to GPS and COLOR parameters. This value is applied AFTER it matches + * the original VISUAL_THRESHOLD. + * + * COLOUR_THRESHOLD_BIAS determines how important colour similarity is in + * comparison to GPS and visual/physical parameters. This value is applied AFTER it matches + * the original COLOUR_THRESHOLD. + */ + double THRESHOLD[4] = {0.1,0.6,0.9,0.5}; double THRESHOLD_BIAS[4] = + {0.5,0.1,0,0}; - void cleanup(); public: + /* + * getInstance() returns the singleton instance of this class. If it is not + * instantiated it is initialized. + * @return the singleton TargetAnalyzer + */ + static TargetAnalyzer * getInstance(){ + if (!analyzer){ + analyzer = new TargetAnalyzer; + } + return analyzer; + } + + /* + * @brief AlgorithmNum enumerates a variety of settings for determining + * the variance and significance of GPS, CONTOUR and COLOUR correlation. + */ + enum AlgorithmNum{ + GPS = 0, + CONTOUR = 1, + COLOUR = 2, + MATCH = 3, + }; - static const TargetAnalyzer * getInstance(); + /* + * analyze_pixelobject(PixelObject* po) is the entry function into this + * module, it initializes the comparisons that are made for each + * PixelObject. + * @param po the PixelObject that as being analyzed. + */ + void analyze_pixelobject(PixelObject* po); + + /* + * @brief get_threshold() is a getter for the threshold parameter of a + * particular analysis algorithm + * @param an the algorithm for which the threshold settings are + * requested for, as defined by enum AlgorithmNum + * @return the threshold setting for the particular algorithm + */ + double get_threshold(AlgorithmNum an); + + /* + * @brief get_threshold_bias() is a getter for the threshold_bias + * parameter of a particular analysis algorithm. + * @param an the algorithm for which the threshold_bias settings are + * requested for, as defined by enum AlgorithmNum + * @return the threshold_bias setting for the particular algorithm + */ + double get_threshold_bias(AlgorithmNum an); + + /* + * @brief set_threshold() is a setter for the threshold parameter of a + * particular analysis algorithm + * @param an the algorithm for which the threshold settings are + * requested for, as defined by enum AlgorithmNum + * @param value the threshold setting for the particular algorithm + */ + void set_threshold(AlgorithmNum an, double value); + + /* + * @brief set_threshold_bias() is a getter for the threshold_bias + * parameter of a particular analysis algorithm. + * @param an the algorithm for which the threshold_bias settings are + * requested for, as defined by enum AlgorithmNum + * @param value the threshold_bias setting for the particular algorithm + */ + void set_threshold_bias(AlgorithmNum an, double value); + /* + * getGPS(...) calculates the GPS coordinates (latitude, longitude) of a + * specific point in a frame. + * @param point the point at which the latitude and longitude should be + * determined at. + * @param cameraAlpha the alpha angle of the lens of the camera. This + * defines how much can be seen and what effect altitude has on the image + * scaling. + * @param f the frame for which the point is calculated for. + * @return A Point, where the first value is the latitude and the second is + * the longitude. + */ + int getGPS(cv::Point2d point, cv::Point2d cameraAlpha,Frame* f, cv::Point2d* returnResult); + + + /* + * @brief getGPSDistance() calculates the distance between two GPS coordinates in meters. + * @param lat1 the latitude of the first point + * @param lon1 the longitude of the first point + * @param lat2 the latitude of the second point + * @param lon2 the longitude of the second point + * @return the distance in meters between the two GPS coordinates. + */ + double getGPSDistance(double lat1, double lon1, double lat2, double lon2); - /** - * @brief Analyzes a pixeltarget and returns a pointer to the unique target - * it belongs to - * - * @param p PixelTarget to be analyzed - * @param f Frame that the PixelTarget belongs to - * @return pointer to the Target that the PixelTarget belongs to + /* + * @brief getGPSDistance() calculates the distance between two GPS coordinates in meters. + * @param gps1 the gps coordinates of the first point + * @param gps2 the gps coordinates of the second point + * @return the distance in meters between the two GPS coordinates. */ - Target * analyze_targets_in_frame(PixelTarget * p, Frame * f); - - /** - * @brief retrieves a vector containing all of the unique Targets that - * have been indentified. - * The caller of the function should not delete the Targets as they - * will be deleted by the TargetAnalyzer destructor - * - * @return vector containing the unique Targets which have been analyzed + double getGPSDistance(cv::Point2d gps1, cv::Point2d gps2); + + /* + * extract_objects() provides a list of unique objects that have been + * identified, where the non-unique ones have been grouped. + * @return the vector of Objects with identifies the unique ones found. */ - vector get_processed(); + std::vector extract_objects(); + + /* + * get_unique_objects_length() provides the total number of objects that + * have been found within the data acquired. + * @return the integer number of Objects that can be extracted by using + * the extract_objects() function. + */ + int get_unique_objects_length(); }; #endif // TARGET_ANALYZER_H_INCLUDED diff --git a/modules/targetanalysis/src/object_list.cpp b/modules/targetanalysis/src/object_list.cpp new file mode 100644 index 00000000..9637e9ba --- /dev/null +++ b/modules/targetanalysis/src/object_list.cpp @@ -0,0 +1,72 @@ +/** + * @file object_list.h + * @author WARG + * + * @section LICENSE + * + * Copyright (c) 2015-2016, Waterloo Aerial Robotics Group (WARG) + * All rights reserved. + * + * This software is licensed under a modified version of the BSD 3 clause license + * that should have been included with this software in a file called COPYING.txt + * Otherwise it is available at: + * https://raw.githubusercontent.com/UWARG/computer-vision/master/COPYING.txt + */ + + +#include "frame.h" +#include "object.h" +#include "object_list.h" + +ObjectList::ObjectList(){ + listLength = 0; + tail = head; +} + +ObjectList::~ObjectList(){ + int i = 0; + poNode* tempPointer; + while(inext; + delete(tempPointer); + i++; + } +} + +bool ObjectList::addNode(Object* o){ + struct oNode* newNode = new struct oNode; + if (newNode){ + newNode->o = o; + newNode->next = 0; //Nullify pointer (Since there is no next list item) + //Update old node with the new node + head->next = newNode; + //Change head to represent the new node + head = newNode; + //Update list length + listLength++; + return true; + } + + return false; + +} + +bool ObjectList::getGPSDuplicates(){ + //TODO: Consider changing the return type. + //Should return array of probabilities (% error based on known GPS error and + //other parameters) + //After multiplying by the visual probability, a good estimate of whether or + //not it is a duplicate will be known + int i = 0; + while(i < listLength){ + tail->o-> + i++; + } + return true; +} + +//TODO:Add later to compare duplicate targets from a visual perspective +//bool ObjectList::getVisualDuplicates(){ + +//} diff --git a/modules/targetanalysis/src/pixel_object_list.cpp b/modules/targetanalysis/src/pixel_object_list.cpp new file mode 100644 index 00000000..95188acb --- /dev/null +++ b/modules/targetanalysis/src/pixel_object_list.cpp @@ -0,0 +1,325 @@ +/** + * @file pixel_object_list.cpp + * @author WARG + * + * @section LICENSE + * + * Copyright (c) 2015-2017, Waterloo Aerial Robotics Group (WARG) + * All rights reserved. + * + * This software is licensed under a modified version of the BSD 3 clause license + * that should have been included with this software in a file called COPYING.txt + * Otherwise it is available at: + * https://raw.githubusercontent.com/UWARG/computer-vision/master/COPYING.txt + */ + + +#include "frame.h" +#include "object.h" +#include "pixel_object.h" +#include "pixel_object_list.h" +#include "target_analyzer.h" +#include "contour_comparison.h" +#include +#include +#include + + +#include "opencv2/highgui/highgui.hpp" +#include "opencv2/imgproc/imgproc.hpp" + +using namespace boost; +using namespace std; +using namespace cv; + + +PixelObjectList* PixelObjectList::firstInstance = NULL; + +PixelObjectList::~PixelObjectList(){ + int i = 0; + poNode* tempPointer; + while(inext; + delete(tempPointer); + i++; + } +} + +void PixelObjectList::addNode(PixelObject* po){ + //Initialize node + struct poNode* newNode = new struct poNode; + //Initialize object within node + Object* newObject = new Object; + newNode->o = newObject; + + newNode->o->add_pobject(po); + newNode->next = 0; //Nullify pointer (Since there is no next list item) + + //Update old node with the new node + if (head){ //If HEAD has been initialized already + head->next = newNode; + } + else{ //If HEAD hasn't been initialized already + tail = newNode; + } + //Change head to represent the new node + head = newNode; + //Update list length + listLength++; + BOOST_LOG_TRIVIAL(debug) << "Node added. New List Length: " << listLength; +} + +double PixelObjectList::compareNode(PixelObject* po1, Object* o2){ + //Use FLANN here? + //Compare Pixel Objects based on Metadata and visual queues + //Uses Hierarchical Structure: GPS data is most important, followed by + //Contour/Shape, followed by Colour + TargetAnalyzer* ta = TargetAnalyzer::getInstance(); + using TA = TargetAnalyzer; + double gps = compareGPS(po1, o2); + if (gps > ta->get_threshold(TA::GPS)){ + double visual = compareContours(po1, o2); + if (visual > ta->get_threshold(TA::CONTOUR)){ + double colour = compareColour(po1, o2); + if (colour > ta->get_threshold(TA::COLOUR)){ + return (gps + ta->get_threshold_bias(TA::GPS))*(visual + ta->get_threshold_bias(TA::CONTOUR))*(colour + ta->get_threshold_bias(TA::COLOUR)); + } + } + } + return 0; +} + +double PixelObjectList::compareColour(PixelObject* po1, Object* o2){ + cv::Scalar a = po1->get_colour(); + cv::Scalar b = o2->get_colour(); + double* aArr = a.val; + double* bArr = b.val; + + double euclideanDistance = sqrt(pow(aArr[0] - bArr[0],2) + pow(aArr[1] - + bArr[1],2) + pow(aArr[2] - bArr[2],2)); + + return 1 - euclideanDistance/sqrt(3*pow(255,2)); +} + +double PixelObjectList::compareGPS(PixelObject* po1, Object* o2){ + //Calculations assume ideal scenario (no tilt in photos, no distortion) + Frame* f = po1->get_image(); + cv::Point2d result(0,0); + if(!TargetAnalyzer::getInstance()->getGPS(po1->get_centroid(),cv::Point2d(69.5/2,125.3/2)/*getSettings()*/,f,&result)){ + //If there is something wrong with the GPS coordinates, its not a valid + //match + return 0; + } + po1->set_gps_centroid(result); + double distance = TargetAnalyzer::getInstance()->getGPSDistance(result, o2->get_centroid()); + if (distance < 1){ + distance = 1; //Prevent zero division + } + return 1.0/distance; //As you get farther away, probability of the target being the same decreases. +} + + +double PixelObjectList::compareContours(PixelObject* po1, Object* o2){ + //Compare width to length ratio, compare area to perimeter ratio, compare + //square area(widthxlength) to area. + double maxDiff = sqrt(5); + + vector contour1 = po1->get_contour(); + //Length is the larger dimension + cv::Rect r1 = cv::boundingRect(contour1); + double l1 = r1.height; + double w1 = r1.width; + //If length is not the larger dimension, switch the values + if (l1 < w1){ + double temp = l1; + l1 = w1; + w1 = temp; + } + double w_l_ratio1 = w1/l1; + double a_p_ratio1 = po1->get_area()/po1->get_perimeter(); + double sq_a_ratio1 = po1->get_area()/(l1*w1); + + for (PixelObject* po2 : o2->get_pobjects()){ + vector contour2 = po2->get_contour(); + + cv::Rect r2 = cv::boundingRect(contour2); + double l2 = r2.height; + double w2 = r2.width; + if (l2 < w2){ + double temp = l2; + l2 = w2; + w2 = temp; + } + double w_l_ratio2 = w2/l2; + double a_p_ratio2 = po2->get_area()/po2->get_perimeter(); + double sq_a_ratio2 = po2->get_area()/(l2*w2); + + double diff = sqrt(pow((l1 - l2)/l1, 2) + pow((w1 - w2)/w2,2) + + pow((w_l_ratio1 - w_l_ratio2)/w_l_ratio1,2) + pow((a_p_ratio1 - + a_p_ratio2)/a_p_ratio1,2) + pow((sq_a_ratio1 - + sq_a_ratio2)/sq_a_ratio1,2)); //5-dimensions -> Max value is sqrt(5) + + if (diff < maxDiff){ + maxDiff = diff; + } + } + return 1 - maxDiff/sqrt(5); +} + +/*double PixelObjectList::compareContours(PixelObject* po1, Object* o2){ + const vector& poList = o2->get_pobjects(); + + double maxSimilarity = 0; + vector minimumContour; + for (PixelObject* po2 : poList){ + vector v1 = po1->get_contour(); + vector v2 = po2->get_contour(); + Point c1 = po1->get_centroid(); + Point c2 = po2->get_centroid(); + + //TODO: Choose A1, A2, po1, po2 based on the smaller area...saves time on computation + int a1 = po1->get_area(); + int a2 = po2->get_area(); + + if (a1 == 0 || a2 == 0){ + continue; + } + + //cArea determines size and accuracy of the matching. Should be at + //least 100. + double areaScale1 = (double)COMPARE_AREA/a1; + double areaScale2 = (double)COMPARE_AREA/a2; + + //Rotate different Angles + int numIntervals = 36; //Should vary based on computation time requirements + vector> v2Mod; + vector v1Mod; + + //Center the contour at (0,0) + for (Point i : v1){ + v1Mod.push_back(areaScale1*(i - c1)); + } + + //Center the contour at (0,0) and rotate it and scale it + for (int interval = 0; interval < numIntervals; interval++){ + double theta = 2*M_PI/numIntervals * interval; + BOOST_LOG_TRIVIAL(debug) << "theta: " << theta << ", aS2: " << + areaScale2 << ", C2: " << c2; + + vector tempVec; + for (Point i : v2){ + tempVec.push_back(Point(areaScale2*cos(theta)*(i.x-c2.x) - + areaScale2*sin(theta)*(i.y-c2.y), + areaScale2*sin(theta)*(i.x-c2.x) + areaScale2*cos(theta)*(i.y-c2.y))); + } + v2Mod.push_back(tempVec); + } + + float radius1, radius2; + Point2f center1, center2; + minEnclosingCircle(v1Mod,center1,radius1); + minEnclosingCircle(v2Mod[0],center2,radius2); + //If areas are similar, check size. If its way off (factor of 2), move onto next iteration. + if (abs(radius1 - radius2) > radius1||abs(radius1 - radius2) > radius2){ + //Skip analysis to save computation time + continue; + } + + for (int i = 0; i < numIntervals; i++){ + //Reposition the contours for Mat operations (in the positive quadrant) + Point matReference(radius1-center1.x > radius2-center2.x?radius1-center1.x:radius2-center2.x, radius1-center1.y > radius2-center2.y?radius1-center1.y:radius2-center2.y); + for (int j = 0; j < v1Mod.size();j++){ + v1Mod[j] = v1Mod[j] + matReference; + } + for (int j = 0; j < v2Mod[i].size(); j++){ + v2Mod[i][j] = v2Mod[i][j] + matReference; + } + vector> contoursWrapperA, contoursWrapperB; + contoursWrapperA.push_back(v1Mod); + contoursWrapperB.push_back(v2Mod[i]); + + //No point comparing contours if there is only one. + if (listLength >= 1){ + double sim = compare_contours(contoursWrapperA, contoursWrapperB); + if (maxSimilarity < sim){ + BOOST_LOG_TRIVIAL(debug) << "Found new optimized position: " << sim; + maxSimilarity = sim; + minimumContour = v2Mod[i]; + + */ /*Visualization + if (sim > 0.7){ + Mat drawing = Mat::zeros(matReference.x*2, matReference.y*2, CV_8UC3); + + drawContours(drawing, contoursWrapperA, 0, cv::Scalar(255,0,255), FILLED); + drawContours(drawing, contoursWrapperB, 0, cv::Scalar(0,255,255), FILLED); + + namedWindow( "Duplicate Contours", CV_WINDOW_AUTOSIZE ); + imshow( "Duplicate Contours", drawing ); + waitKey(0); + }*//* + } + } + } + } + return maxSimilarity; +}*/ + +void PixelObjectList::addAndCompare(PixelObject* po){ + //Iterate over list + int i = 0; + struct comparitor listMatch[listLength]; + double maxSimilarity = 0; + int iMax = 0; + + poNode* tempPointer; + tempPointer = tail; + BOOST_LOG_TRIVIAL(info) << "List Length: " << listLength; + while(io); + listMatch[i].node = tempPointer; + + //Update the best match if a better one is available + if (listMatch[i].similarity > maxSimilarity){ + maxSimilarity = listMatch[i].similarity; + iMax = i; + } + i++; + tempPointer = tempPointer->next; + } + //Is the best match good enough - FUZZY LOGIC + if (maxSimilarity >= TargetAnalyzer::getInstance()->get_threshold(TargetAnalyzer::MATCH)){ + //The add_pobject function should also recalculate all parameters of the + //object. + listMatch[iMax].node->o->add_pobject(po); + } + else{ + //Add the node to the end of the list for future comparisons + //Note, that objects that have already matched are not included for + //future comparisons. This needs to be tested in the field. + addNode(po); + } + +} + +int PixelObjectList::getListLength(){ + return listLength; +} + +Object* PixelObjectList::getObject(int index){ + struct poNode* node = tail; + for (int i = 0; i < index; i++){ + node = node->next; + } + return node->o; +} + +void PixelObjectList::getObjects(vector* v){ + struct poNode* node = tail; + for (int i = 0; i < listLength; i++){ + v->push_back(node->o); + node = node->next; + } +} diff --git a/modules/targetanalysis/src/target_analyzer.cpp b/modules/targetanalysis/src/target_analyzer.cpp index 4981940b..5756fbc6 100644 --- a/modules/targetanalysis/src/target_analyzer.cpp +++ b/modules/targetanalysis/src/target_analyzer.cpp @@ -1,45 +1,161 @@ -/* - This file is part of WARG's computer-vision - - Copyright (c) 2015, Waterloo Aerial Robotics Group (WARG) - All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are met: - 1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - 3. Usage of this code MUST be explicitly referenced to WARG and this code - cannot be used in any competition against WARG. - 4. Neither the name of the WARG nor the names of its contributors may be used - to endorse or promote products derived from this software without specific - prior written permission. - - THIS SOFTWARE IS PROVIDED BY WARG ''AS IS'' AND ANY - EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - DISCLAIMED. IN NO EVENT SHALL WARG BE LIABLE FOR ANY - DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*/ - -/*TODO: First thing to do; -1) Convert Pixel location into actual location; Pixel Area into actual area -2) Add classfiers for targets using: -Area, Centroid, Perimeter, Colour, Error, contour, edge magnitude -Evaluate each to classify as each target and apply percentage to -3) -*/ +/** + * @file target_analyzer.cpp + * @author WARG + * + * @section LICENSE + * + * Copyright (c) 2015-2017, Waterloo Aerial Robotics Group (WARG) + * All rights reserved. + * + * This software is licensed under a modified version of the BSD 3 clause license + * that should have been included with this software in a file called COPYING.txt + * Otherwise it is available at: + * https://raw.githubusercontent.com/UWARG/computer-vision/master/COPYING.txt + */ #include "frame.h" #include "pixel_object.h" +#include "pixel_object_list.h" +#include "object_list.h" +#include "target_analyzer.h" +#include "target.h" -void analyze_targets_in_frame(Frame * f, PixelObject t){ +#include +#include +#include +using namespace std; + +TargetAnalyzer* TargetAnalyzer::analyzer = NULL; +vector mostRecentObjectList; + +void TargetAnalyzer::analyze_pixelobject(PixelObject* po){ + PixelObjectList* pol = PixelObjectList::getInstance(); + pol->addAndCompare(po); + BOOST_LOG_TRIVIAL(trace) << "End of Analysis"; +} + + +vector TargetAnalyzer::extract_objects(){ + PixelObjectList* pol = PixelObjectList::getInstance(); + pol->getObjects(&mostRecentObjectList); + return mostRecentObjectList; } + +int TargetAnalyzer::get_unique_objects_length(){ + PixelObjectList* pol = PixelObjectList::getInstance(); + return pol->getListLength(); +} + +double TargetAnalyzer::get_threshold(AlgorithmNum an){ + return THRESHOLD[an]; +} + +double TargetAnalyzer::get_threshold_bias(AlgorithmNum an){ + return THRESHOLD_BIAS[an]; +} + +void TargetAnalyzer::set_threshold(AlgorithmNum an, double value){ + THRESHOLD[an] = value; +} + +void TargetAnalyzer::set_threshold_bias(AlgorithmNum an, double value){ + THRESHOLD_BIAS[an] = value; +} + + +//Based on the GPS location of the image, calculates the +//GPS location of a certain pixel in the image. +int TargetAnalyzer::getGPS(cv::Point2d point, cv::Point2d cameraAlpha, +Frame* f, cv::Point2d* returnResult){ + const Metadata* m = f->get_metadata(); + cv::Mat img = f->get_img(); + int h = img.cols; + int w = img.rows; + + if (w <= 0 || h <= 0){ + return 0; + } + + cv::Point2d imgCenter(w/2, h/2); + + //(0,0) is in the center of the image + cv::Point2d biasedPoint = point - imgCenter; + + double altitude = m->altitude; + double heading = m->heading; + double latitude = m->lat; + double longitude = m->lon; + + //Note: The cameraAlpha value represents the half angle of the x and y//direction of the image. + double cameraXEdge = altitude / tan(DEG2RAD(90 - cameraAlpha.x)); //meters from center of photo to edge + double cameraYEdge = altitude / tan(DEG2RAD(90 - cameraAlpha.y)); //meters from center of photo to edge + + //Rotation Matrix - Heading + //Note: The '-heading' compensates for the fact that directional heading is + //a clockwise quantity, but cos(theta) assumes theta is a counterclockwise + //quantity. + double realXEdge = cos(DEG2RAD(-heading)) * cameraXEdge - sin(DEG2RAD(-heading)) * + cameraYEdge; + double realYEdge = sin(DEG2RAD(-heading)) * cameraXEdge + cos(DEG2RAD(-heading)) * + cameraYEdge; + + double realX = cos(DEG2RAD(-heading)) * biasedPoint.x/(w/2)*cameraXEdge - sin(DEG2RAD(-heading)) * + biasedPoint.y/(h/2)*cameraYEdge; + double realY = sin(DEG2RAD(-heading)) * biasedPoint.x/(w/2)*cameraXEdge + cos(DEG2RAD(-heading)) * + biasedPoint.y/(h/2)*cameraYEdge; + + double lon = RAD2DEG(realX/EARTH_RADIUS)/cos(DEG2RAD(latitude)) + longitude; + double lat = RAD2DEG(realY/EARTH_RADIUS) + latitude; + + double unitX = realXEdge/img.cols; + double unitY = realYEdge/img.rows; + + f->set_pixel_distance(unitX,unitY); + *returnResult = cv::Point2d(lat,lon); + return 1; +} + + + +//TODO: The image data fed into this function should have the camera distortion +//corrected for. +//TODO: Add compensation for roll and pitch angles. This should skew the +//photo/gps grid. The lens profile may have a big effect here. See previous +//todo. + +double TargetAnalyzer::getGPSDistance(double lat1, double lon1, double lat2, double lon2){ + double dLat = DEG2RAD(lat2 - lat1); + double dLon = DEG2RAD(lon2 - lon1); + + float a = sin(dLat / 2) * sin(dLat / 2) + cos(DEG2RAD(lat1)) * cos(DEG2RAD(lat2)) * sin(dLon / 2) * sin(dLon / 2); + + /*if ((dLat >= 0 && dLon >=0)||(dLat < 0 && dLon < 0)){*/ + return EARTH_RADIUS * (2 * atan2(sqrt(a),sqrt(1 - a))); + /*}*/ + /*else { + return EARTH_RADIUS * (2 * atan2(sqrt(a),sqrt(1 - a))) * -1; + }*/ + +} + +double TargetAnalyzer::getGPSDistance(cv::Point2d gps1, cv::Point2d gps2){ + return getGPSDistance(gps1.x, gps1.y, gps2.x, gps2.y); +} + //For future thought: GPS GEOLOCATION IS THE LEAST RELIABLE PART, WHAT IS THE ACTUAL + //HEADING??? We use GPS Heading, but the wind can alter this by upto + //30degreesC + //This code assumes the top of the image is also pointing towards the heading + //measured by the aircraft + + //Run functions to determine probability of the image displaying a duplicate based on visual, telemetry data. + //Things to consider for the future: + //*TargetList* + //-Feature Matching + //-GPS coordinate matching + //-Area Matching + //-Perimeter Matching + //-Contour Matching + //-Colour Matching + //-Consider expected error + diff --git a/modules/targetanalysis/src/target_loader.cpp b/modules/targetanalysis/src/target_loader.cpp index 94286659..1062ecfc 100644 --- a/modules/targetanalysis/src/target_loader.cpp +++ b/modules/targetanalysis/src/target_loader.cpp @@ -80,6 +80,7 @@ TargetParameters targetLoader:getParameters(){ return 0; } */ + TargetLoader::TargetLoader(const char* file){ if (readFile(file)){ std::string data = std::string(rawData); @@ -87,6 +88,23 @@ TargetLoader::TargetLoader(const char* file){ } } +/* +PixelObject* TargetLoader::getPixelObject(){ + using boost::property_tree::ptree; + ptree::const_iterator end = jsonParameters.end(); + for (ptree::const_iterator it = jsonParameters.begin(); it != end; ++it) { + if (it->first == "contour"){ + std::cout << it->first << ": " << it->second.get_value() << std::endl; + recursivePrint(it->second); + } + } + PixelObject* po = new PixelObject(cv::Mat & crop, + std::vector & contour, cv::Point2d centroid, double area, double + perimeter, cv::Scalar colour, cv::Point2d error, double errorAngle); + + return po; +}*/ + bool TargetLoader::readFile(const char* fileLocation){ ifstream jFile(fileLocation, ifstream::in | ios::ate); diff --git a/modules/targetanalysis/test/CMakeLists.txt b/modules/targetanalysis/test/CMakeLists.txt index d597a3e4..1340ba9b 100644 --- a/modules/targetanalysis/test/CMakeLists.txt +++ b/modules/targetanalysis/test/CMakeLists.txt @@ -1,10 +1,20 @@ enable_testing() +include_directories(../include ../../core/include ../../targetid/include +../../imgimport/include) if(Boost_FOUND AND OpenCV_FOUND) ADD_DEFINITIONS("-DBOOST_LOG_DYN_LINK") add_executable(target_loader_test test.cpp) + add_executable(unique_po_test unique_po_test.cpp) + add_executable(gps_point_test gps_point_test.cpp) target_link_libraries(target_loader_test ${OpenCV_LIBS} ${Boost_LIBRARIES} Core TargetAnalysis) + target_link_libraries(unique_po_test ${OpenCV_LIBS} ${Boost_LIBRARIES} Core TargetAnalysis TargetIdentification) + target_link_libraries(gps_point_test ${OpenCV_LIBS} ${Boost_LIBRARIES} Core + TargetAnalysis ImageImport) # Tests add_test("SimpleLoad" target_loader_test "${TESTDATA_DIR}/sample.json" "Simple Load" --log_format=XML --log_sink=TEST_LOADER.xml --log_level=all --report_level=no) + add_test("UniquePOTest" unique_po_test "${TESTDATA_DIR}/photos" "Pixel Object Uniqueness" --log_format=XML --log_sink=TEST_LOADER.xml --log_level=all --report_level=no) + add_test("GPSPointTest" gps_point_test "${TESTDATA_DIR}/photos" "GPS Point Test" --log_format=XML --log_sink=TEST_LOADER.xml --log_level=all --report_level=no) + endif() diff --git a/modules/targetanalysis/test/gps_point_test.cpp b/modules/targetanalysis/test/gps_point_test.cpp new file mode 100644 index 00000000..378446de --- /dev/null +++ b/modules/targetanalysis/test/gps_point_test.cpp @@ -0,0 +1,260 @@ +/** + * @file gps_point_test.cpp + * @author WARG + * + * @section LICENSE + * + * Copyright (c) 2015-2017, Waterloo Aerial Robotics Group (WARG) + * All rights reserved. + * + * This software is licensed under a modified version of the BSD 3 clause license + * that should have been included with this software in a file called COPYING.txt + * Otherwise it is available at: + * https://raw.githubusercontent.com/UWARG/computer-vision/master/COPYING.txt + */ + +#define BOOST_TEST_DYN_LINK +#define BOOST_TEST_MODULE TargetAnalysis + +#include +#include +#include +#include +#include +#include +#include + +#include "pixel_object_list.h" +#include "target_analyzer.h" +#include "target_identifier.h" +#include "canny_contour_creator.h" +#include "k_means_filter.h" +#include "frame.h" +#include "metadata_reader.h" +#include "metadata_input.h" + +#include +#include +#include + +#include + +using namespace std; +using namespace boost; +using namespace cv; + +namespace logging = boost::log; +using namespace boost::filesystem; + +bool has_suffix(const std::string &str, const std::string &suffix) +{ + return str.size() >= suffix.size() && str.compare(str.size() - suffix.size(), suffix.size(), suffix) == 0; +} + + +BOOST_AUTO_TEST_CASE(GPSHighAltitudePointTest){ + + if(boost::unit_test::framework::master_test_suite().argc <= 1) { + BOOST_ERROR("Invalid number of arguments"); + } + + string root_path = boost::unit_test::framework::master_test_suite().argv[1]; + string description = boost::unit_test::framework::master_test_suite().argv[2]; + + //Read directory + BOOST_REQUIRE( filesystem::exists( root_path ) ); + directory_iterator end_itr; // default construction yields past-the-end iterator + int fileCount = std::count_if(directory_iterator(root_path),directory_iterator(),static_cast(is_regular_file)); + + + int numImage = 0; + int numPixelObjects = 0; + PixelObject* pointerList[fileCount]; + KMeansFilter * filter = new KMeansFilter(); + CannyContourCreator * ccreator = new CannyContourCreator(); + + double longitude, latitude, altitude; + + for ( directory_iterator itr( root_path ); itr != end_itr; ++itr ){ + if ( is_regular_file(itr->path()) ){ + string current_file = itr->path().string(); + if (has_suffix(current_file, ".jpg") || has_suffix(current_file, + ".jpeg")){ + + + //Manually generate a PixelObject + MetadataInput* mi = new MetadataInput(); + string filename = root_path + "/../test_csv.csv"; + MetadataReader mr(*mi,filename); + cv::Mat* img = new cv::Mat(imread(current_file,CV_LOAD_IMAGE_UNCHANGED)); + //Skip first zero row + mi->next_metadata(); + const Metadata m = mi->get_metadata(220649); + Camera cam = Camera::TestCamera(); + Frame* f = new Frame(img, current_file, m, cam); + + longitude = m.lon; + latitude = m.lat; + altitude = m.altitude; + BOOST_LOG_TRIVIAL(debug) << "Frame->Longitude: " << m.lon << + "; Latitude: " << m.lat << "; Altitude: " << m.altitude << endl; + + //Generate Contour + ObjectDetector detector(filter, ccreator); + detector.process_frame(f); + vector results; + PixelObject* o; + for (PixelObject * o : f->get_objects()) { + + vector contour = results; + + //POList + pointerList[numPixelObjects++] = o; + } + numImage++; + break; + } + + } + } + + //Inject into GPS Location Function + PixelObjectList* po = PixelObjectList::getInstance(); + cv::Point2d cameraAlpha(69.5/2,118.2/2); + cv::Point2d* geoLocation[numPixelObjects]; + for (int i = 0; i < numPixelObjects; i++){ + BOOST_LOG_TRIVIAL(trace) << "Processing PixelObject: " << i; + geoLocation[i] = new cv::Point2d(); + TargetAnalyzer* ta = TargetAnalyzer::getInstance(); + BOOST_CHECK(ta->getGPS(pointerList[i]->get_centroid(),cameraAlpha,pointerList[i]->get_image(),geoLocation[i])==1); + double gpsDistance = ta->getGPSDistance(latitude, longitude, + geoLocation[i]->x, geoLocation[i]->y); + + double dLat = abs(latitude - geoLocation[i]->x); + double dLon = abs(longitude - geoLocation[i]->y); + + BOOST_CHECK_MESSAGE(dLat > 1e-4 && dLat < 1e-2, "Calculated Latitude is not within acceptable limits: " + to_string(dLat)); + BOOST_CHECK_MESSAGE(dLon > 1e-4 && dLon < 1e-2, "Calculated Longitude is not within acceptable limits: " + to_string(dLon)); + + BOOST_CHECK_MESSAGE(gpsDistance < 167 && gpsDistance > 83, "Distance from center GPS is too large or too small."); + + BOOST_LOG_TRIVIAL(info) << "New GPS Coordinates: " << *geoLocation[i]; + BOOST_LOG_TRIVIAL(info) << "dlat: " << dLat << " dlon: " << dLon; + BOOST_LOG_TRIVIAL(info) << "Distance from center of photo: " << gpsDistance; + + } + + //Cleanup + delete filter; + delete ccreator; + for (int i = 0; i < numPixelObjects; i++){ + delete geoLocation[i]; + } +} + +BOOST_AUTO_TEST_CASE(GPSLowAltitudePointTest){ + if(boost::unit_test::framework::master_test_suite().argc <= 1) { + BOOST_ERROR("Invalid number of arguments"); + } + + string root_path = boost::unit_test::framework::master_test_suite().argv[1]; + string description = boost::unit_test::framework::master_test_suite().argv[2]; + + //Read directory + BOOST_REQUIRE( filesystem::exists( root_path ) ); + directory_iterator end_itr; // default construction yields past-the-end iterator + int fileCount = std::count_if(directory_iterator(root_path),directory_iterator(),static_cast(is_regular_file)); + + + int numImage = 0; + int numPixelObjects = 0; + PixelObject* pointerList[fileCount]; + KMeansFilter * filter = new KMeansFilter(); + CannyContourCreator * ccreator = new CannyContourCreator(); + double longitude, latitude, altitude; + + for ( directory_iterator itr( root_path ); itr != end_itr; ++itr ){ + if ( is_regular_file(itr->path()) ){ + string current_file = itr->path().string(); + if (has_suffix(current_file, ".jpg") || has_suffix(current_file, + ".jpeg")){ + + + //Manually generate a PixelObject + MetadataInput* mi = new MetadataInput(); + string filename = root_path + "/../test_csv.csv"; + MetadataReader mr(*mi,filename); + cv::Mat* img = new cv::Mat(imread(current_file,CV_LOAD_IMAGE_UNCHANGED)); + //Skip first zero row + mi->next_metadata(); + const Metadata m = mi->next_metadata(); + Camera cam = Camera::TestCamera(); + Frame* f = new Frame(img, current_file, m, cam); + + longitude = m.lon; + latitude = m.lat; + altitude = m.altitude; + BOOST_LOG_TRIVIAL(debug) << "Frame->Longitude: " << m.lon << + "; Latitude: " << m.lat << "; Altitude: " << m.altitude << endl; + + //Generate Contour + ObjectDetector detector(filter, ccreator); + detector.process_frame(f); + vector results; + for (PixelObject * o : f->get_objects()) { + + /*vector contour = o->get_contour(); + vector> contourWrapper; + contourWrapper.push_back(contour); + Mat drawing = Mat::zeros(2000,2000, CV_8UC3); + drawContours(drawing,contourWrapper,0,cv::Scalar(255,0,255),FILLED); + namedWindow("Contours", CV_WINDOW_AUTOSIZE); + imshow("Contours",drawing); + waitKey(0);*/ + + //POList + pointerList[numPixelObjects++] = o; + } + numImage++; + break; + } + + } + } + + //Inject into GPS Location Function + PixelObjectList* po = PixelObjectList::getInstance(); + cv::Point2d cameraAlpha(69.5/2,118.2/2); + cv::Point2d* geoLocation[numPixelObjects]; + for (int i = 0; i < numPixelObjects; i++){ + BOOST_LOG_TRIVIAL(trace) << "Processing PixelObject: " << i; + geoLocation[i] = new cv::Point2d(); + TargetAnalyzer* ta = TargetAnalyzer::getInstance(); + BOOST_CHECK(ta->getGPS(pointerList[i]->get_centroid(),cameraAlpha,pointerList[i]->get_image(),geoLocation[i])==1); + + double dLat = abs(latitude - geoLocation[i]->x); + double dLon = abs(longitude - geoLocation[i]->y); + double gpsDistance = ta->getGPSDistance(latitude, longitude, + geoLocation[i]->x, geoLocation[i]->y); + + //Low altitude image should have near zero displacement on imagery + BOOST_CHECK_MESSAGE(dLat < 1e-4, "Calculated Latitude is not within acceptable limits: " + to_string(dLat)); + BOOST_CHECK_MESSAGE(dLon < 1e-4, "Calculated Longitude is not within acceptable limits: " + to_string(dLon)); + + BOOST_CHECK_MESSAGE(gpsDistance < 5, "Distance from center GPS is too large."); + + BOOST_LOG_TRIVIAL(info) << "New GPS Coordinates: " << *geoLocation[i]; + BOOST_LOG_TRIVIAL(info) << "dlat: " << dLat << " dlon: " << dLon; + BOOST_LOG_TRIVIAL(info) << "Distance from center of photo: " << gpsDistance; + + } + + //Cleanup + delete filter; + delete ccreator; + for (int i = 0; i < numPixelObjects; i++){ + delete geoLocation[i]; + } +} diff --git a/modules/targetanalysis/test/test.cpp b/modules/targetanalysis/test/test.cpp index c65be6c3..ed160ee4 100644 --- a/modules/targetanalysis/test/test.cpp +++ b/modules/targetanalysis/test/test.cpp @@ -1,33 +1,18 @@ -/* - This file is part of WARG's computer-vision +/** + * @file test.cpp + * @author WARG + * + * @section LICENSE + * + * Copyright (c) 2015-2017, Waterloo Aerial Robotics Group (WARG) + * All rights reserved. + * + * This software is licensed under a modified version of the BSD 3 clause license + * that should have been included with this software in a file called COPYING.txt + * Otherwise it is available at: + * https://raw.githubusercontent.com/UWARG/computer-vision/master/COPYING.txt + */ - Copyright (c) 2015, Waterloo Aerial Robotics Group (WARG) - All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are met: - 1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - 3. Usage of this code MUST be explicitly referenced to WARG and this code - cannot be used in any competition against WARG. - 4. Neither the name of the WARG nor the names of its contributors may be used - to endorse or promote products derived from this software without specific - prior written permission. - - THIS SOFTWARE IS PROVIDED BY WARG ''AS IS'' AND ANY - EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - DISCLAIMED. IN NO EVENT SHALL WARG BE LIABLE FOR ANY - DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*/ #define BOOST_TEST_DYN_LINK #define BOOST_TEST_MODULE TargetAnalysis diff --git a/modules/targetanalysis/test/unique_po_test.cpp b/modules/targetanalysis/test/unique_po_test.cpp new file mode 100644 index 00000000..ccb58ff4 --- /dev/null +++ b/modules/targetanalysis/test/unique_po_test.cpp @@ -0,0 +1,144 @@ +/** + * @file unique_po_test.cpp + * @author WARG + * + * @section LICENSE + * + * Copyright (c) 2015-2017, Waterloo Aerial Robotics Group (WARG) + * All rights reserved. + * + * This software is licensed under a modified version of the BSD 3 clause license + * that should have been included with this software in a file called COPYING.txt + * Otherwise it is available at: + * https://raw.githubusercontent.com/UWARG/computer-vision/master/COPYING.txt + */ + +#define BOOST_TEST_DYN_LINK +#define BOOST_TEST_MODULE TargetAnalysis + +#include +#include +#include +#include +#include +#include +#include + +#include "target_analyzer.h" +#include "target_identifier.h" +#include "canny_contour_creator.h" +#include "k_means_filter.h" +#include "frame.h" + +#include +#include +#include + +#define PIXEL_OBJECTS_PER_IMAGE 7 + +using namespace std; +using namespace boost; +using namespace cv; + +namespace logging = boost::log; +using namespace boost::filesystem; + +bool has_suffix(const std::string &str, const std::string &suffix) +{ + return str.size() >= suffix.size() && str.compare(str.size() - suffix.size(), suffix.size(), suffix) == 0; +} + + +BOOST_AUTO_TEST_CASE(UniquePOTest){ + if(boost::unit_test::framework::master_test_suite().argc <= 1) { + BOOST_ERROR("Invalid number of arguments"); + } + + string root_path = boost::unit_test::framework::master_test_suite().argv[1]; + string description = boost::unit_test::framework::master_test_suite().argv[2]; + + //Read directory + BOOST_REQUIRE( filesystem::exists( root_path ) ); + directory_iterator end_itr; // default construction yields past-the-end iterator + int fileCount = std::count_if(directory_iterator(root_path),directory_iterator(),static_cast(is_regular_file)); + + + int numImage = 0; + int numPixelObjects = 0; + PixelObject* pointerList[fileCount * PIXEL_OBJECTS_PER_IMAGE]; + KMeansFilter * filter = new KMeansFilter(); + CannyContourCreator * ccreator = new CannyContourCreator(); + + for ( directory_iterator itr( root_path ); itr != end_itr; ++itr ){ + if ( is_regular_file(itr->path()) ){ + string current_file = itr->path().string(); + if (has_suffix(current_file, ".jpg") || has_suffix(current_file, + ".jpeg")){ + + //Manually generate a PixelObject + cv::Mat* img = new cv::Mat(imread(current_file,CV_LOAD_IMAGE_UNCHANGED)); + Metadata meta; + meta.lat=49.9118449998;meta.lon=-98.26867;meta.time=215410.5+numImage;meta.pitch=-6.8860778809;meta.roll=-10.4903907776;meta.pitchRate=-0.0558875017;meta.rollRate=0.0036246777;meta.yawRate=0.0214070547;meta.altitude=-1.375;meta.heading=175; + Camera cam = Camera::TestCamera(); + Frame* f = new Frame(img, current_file, meta,cam); + + //Generate Contour + ObjectDetector detector(filter, ccreator); + detector.process_frame(f); + vector results; + for (PixelObject * o : f->get_objects()) { + + vector contour = results;//{cv::Point(0,0),cv::Point(0,1),cv::Point(1,1),cv::Point(1,0)}; + /* + cv::Point2d centroid(0,0); + double area = 0; + double perimeter = 0; + cv::Scalar colour(127,127,127); + cv::Point2d error(0,0); + double errorAngle = 0; + */ + //BOOST_LOG_TRIVIAL(trace) << "Centroid: " << o->get_centroid(); + + //POList + pointerList[numPixelObjects++] = o; //new PixelObject(img,contour,centroid,area,perimeter,colour,error,errorAngle); + } + numImage++; + } + + } + } + BOOST_LOG_TRIVIAL(debug) << "Images in test: " << numImage; + BOOST_LOG_TRIVIAL(debug) << "PixelObjects in test: " << numPixelObjects; + + BOOST_WARN_MESSAGE(numPixelObjects == 75, "Unexpected number of pixel objects, has the target id algorithm changed? Are there more images than expected?"); //If the number of test photos has changed, this integration test may become invalid + BOOST_WARN_MESSAGE(numImage == 24, "Image repository has been updated. Expect inconsistent test data."); + + + //Inject into analysis + TargetAnalyzer* ta = TargetAnalyzer::getInstance(); + for (int i = 0; i < numPixelObjects; i++){ + BOOST_LOG_TRIVIAL(trace) << "Processing Image: " << i; + ta->analyze_pixelobject(pointerList[i]); + } + + + //NEGATIVE CHECK - Check how many different targets there are + int listLength = ta->get_unique_objects_length(); + BOOST_CHECK(listLength == 42); + + //POSITIVE CHECK - Check how many targets got grouped together and how many + //in each according to the predefined images => There should be 3 grouped + //into a single object. + vector list = ta->extract_objects(); + int i = 0; + for (Object* o : list){ + int size = o->get_pobjects().size(); + BOOST_LOG_TRIVIAL(debug) << i << ": " << size; + if (size > 1){ + //This is a little silly, something better should be made. + BOOST_CHECK_MESSAGE(size <= 7, "Unexpected size"); + } + i++; + } +} diff --git a/modules/targetid/src/object_detector.cpp b/modules/targetid/src/object_detector.cpp index deea5933..5f8d0644 100644 --- a/modules/targetid/src/object_detector.cpp +++ b/modules/targetid/src/object_detector.cpp @@ -69,6 +69,14 @@ void ObjectDetector::process_frame(Frame * f){ Mat mask = Mat::zeros(f->get_img().size(), CV_8UC1); drawContours(mask, vector >({contour}), 0, Scalar(255), CV_FILLED); colour = mean(f->get_img(), mask); + Rect bounds = boundingRect(contour); + try { + crop = f->get_img()(Range(bounds.y - bounds.height * 0.2, bounds.y + 1.2 * bounds.height), Range(bounds.x - bounds.width * 0.2, bounds.x + 1.2 * bounds.width)); + } catch (cv::Exception *e) { + + } catch (cv::Exception e) { + + } PixelObject * p = new PixelObject(crop, contour, centroid, area, perimeter, colour, error, errorAngle); f->add_object(p); diff --git a/testdata/photos/G0022791.JPG b/testdata/photos/G0022791.JPG new file mode 100755 index 00000000..fa447120 Binary files /dev/null and b/testdata/photos/G0022791.JPG differ diff --git a/testdata/photos/G0022793.JPG b/testdata/photos/G0022793.JPG new file mode 100755 index 00000000..b8997519 Binary files /dev/null and b/testdata/photos/G0022793.JPG differ diff --git a/testdata/photos/G0022794.JPG b/testdata/photos/G0022794.JPG new file mode 100755 index 00000000..d375c999 Binary files /dev/null and b/testdata/photos/G0022794.JPG differ diff --git a/testdata/photos/G0022796.JPG b/testdata/photos/G0022796.JPG new file mode 100755 index 00000000..0801d425 Binary files /dev/null and b/testdata/photos/G0022796.JPG differ