{

std::cout << "The sample requires nonfree module that is not available in your OpenCV distribution." << std::endl;

return -1;

{

cout << "\nThis program shows how to read in, train on and produce test results for the PASCAL VOC (Visual Object Challenge) data. \n"

<< "It shows how to use detectors, descriptors and recognition methods \n"

"Using OpenCV version %s\n" << CV_VERSION << "\n"

<< "Call: \n"

<< "Format:\n ./" << argv[0] << " [VOC path] [result directory] \n"

<< " or: \n"

<< " ./" << argv[0] << " [VOC path] [result directory] [feature detector] [descriptor extractor] [descriptor matcher] \n"

<< "\n"

<< "Input parameters: \n"

<< "[VOC path] Path to Pascal VOC data (e.g. /home/my/VOCdevkit/VOC2010). Note: VOC2007-VOC2010 are supported. \n"

<< "[result directory] Path to result diractory. Following folders will be created in [result directory]: \n"

<< " bowImageDescriptors - to store image descriptors, \n"

<< " svms - to store trained svms, \n"

<< " plots - to store files for plots creating. \n"

<< "[feature detector] Feature detector name (e.g. SURF, FAST...) - see createFeatureDetector() function in detectors.cpp \n"

<< " Currently 12/2010, this is FAST, STAR, SIFT, SURF, MSER, GFTT, HARRIS \n"

<< "[descriptor extractor] Descriptor extractor name (e.g. SURF, SIFT) - see createDescriptorExtractor() function in descriptors.cpp \n"

<< " Currently 12/2010, this is SURF, OpponentSIFT, SIFT, OpponentSURF, BRIEF \n"

<< "[descriptor matcher] Descriptor matcher name (e.g. BruteForce) - see createDescriptorMatcher() function in matchers.cpp \n"

<< " Currently 12/2010, this is BruteForce, BruteForce-L1, FlannBased, BruteForce-Hamming, BruteForce-HammingLUT \n"

<< "\n";

{

#if defined WIN32 || defined _WIN32

CreateDirectory( dir.c_str(), 0 );

#else

mkdir( dir.c_str(), S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH );

#endif

{

makeDir(rootPath + bowImageDescriptorsDir);

makeDir(rootPath + svmsDir);

makeDir(rootPath + plotsDir);

{

string object_class;

Rect boundingBox;

{

bool difficult;

bool occluded;

bool truncated;

VocPose pose;

{

ObdImage(string p_id, string p_path) : id(p_id), path(p_path) {}

string id;

string path;

{

float score;

int image_idx;

int obj_idx;

bool operator < (const ObdScoreIndexSorter& compare) const {return (score < compare.score);}

{

VocData( const string& vocPath, bool useTestDataset )

{ initVoc( vocPath, useTestDataset ); }

~VocData(){}

/* functions for returning classification/object data for multiple images given an object class */

void getClassImages(const string& obj_class, const ObdDatasetType dataset, vector<ObdImage>& images, vector<char>& object_present);

void getClassObjects(const string& obj_class, const ObdDatasetType dataset, vector<ObdImage>& images, vector<vector<ObdObject> >& objects);

void getClassObjects(const string& obj_class, const ObdDatasetType dataset, vector<ObdImage>& images, vector<vector<ObdObject> >& objects, vector<vector<VocObjectData> >& object_data, vector<VocGT>& ground_truth);

/* functions for returning object data for a single image given an image id */

ObdImage getObjects(const string& id, vector<ObdObject>& objects);

ObdImage getObjects(const string& id, vector<ObdObject>& objects, vector<VocObjectData>& object_data);

ObdImage getObjects(const string& obj_class, const string& id, vector<ObdObject>& objects, vector<VocObjectData>& object_data, VocGT& ground_truth);

/* functions for returning the ground truth (present/absent) for groups of images */

void getClassifierGroundTruth(const string& obj_class, const vector<ObdImage>& images, vector<char>& ground_truth);

void getClassifierGroundTruth(const string& obj_class, const vector<string>& images, vector<char>& ground_truth);

int getDetectorGroundTruth(const string& obj_class, const ObdDatasetType dataset, const vector<ObdImage>& images, const vector<vector<Rect> >& bounding_boxes, const vector<vector<float> >& scores, vector<vector<char> >& ground_truth, vector<vector<char> >& detection_difficult, bool ignore_difficult = true);

/* functions for writing VOC-compatible results files */

void writeClassifierResultsFile(const string& out_dir, const string& obj_class, const ObdDatasetType dataset, const vector<ObdImage>& images, const vector<float>& scores, const int competition = 1, const bool overwrite_ifexists = false);

/* functions for calculating metrics from a set of classification/detection results */

string getResultsFilename(const string& obj_class, const VocTask task, const ObdDatasetType dataset, const int competition = -1, const int number = -1);

void calcClassifierPrecRecall(const string& obj_class, const vector<ObdImage>& images, const vector<float>& scores, vector<float>& precision, vector<float>& recall, float& ap, vector<size_t>& ranking);

void calcClassifierPrecRecall(const string& obj_class, const vector<ObdImage>& images, const vector<float>& scores, vector<float>& precision, vector<float>& recall, float& ap);

void calcClassifierPrecRecall(const string& input_file, vector<float>& precision, vector<float>& recall, float& ap, bool outputRankingFile = false);

/* functions for calculating confusion matrices */

void calcClassifierConfMatRow(const string& obj_class, const vector<ObdImage>& images, const vector<float>& scores, const VocConfCond cond, const float threshold, vector<string>& output_headers, vector<float>& output_values);

void calcDetectorConfMatRow(const string& obj_class, const ObdDatasetType dataset, const vector<ObdImage>& images, const vector<vector<float> >& scores, const vector<vector<Rect> >& bounding_boxes, const VocConfCond cond, const float threshold, vector<string>& output_headers, vector<float>& output_values, bool ignore_difficult = true);

/* functions for outputting gnuplot output files */

void savePrecRecallToGnuplot(const string& output_file, const vector<float>& precision, const vector<float>& recall, const float ap, const string title = string(), const VocPlotType plot_type = CV_VOC_PLOT_SCREEN);

/* functions for reading in result/ground truth files */

void readClassifierGroundTruth(const string& obj_class, const ObdDatasetType dataset, vector<ObdImage>& images, vector<char>& object_present);

void readClassifierResultsFile(const std:: string& input_file, vector<ObdImage>& images, vector<float>& scores);

void readDetectorResultsFile(const string& input_file, vector<ObdImage>& images, vector<vector<float> >& scores, vector<vector<Rect> >& bounding_boxes);

/* functions for getting dataset info */

const vector<string>& getObjectClasses();

string getResultsDirectory();

void initVoc( const string& vocPath, const bool useTestDataset );

void initVoc2007to2010( const string& vocPath, const bool useTestDataset);

void readClassifierGroundTruth(const string& filename, vector<string>& image_codes, vector<char>& object_present);

void readClassifierResultsFile(const string& input_file, vector<string>& image_codes, vector<float>& scores);

void readDetectorResultsFile(const string& input_file, vector<string>& image_codes, vector<vector<float> >& scores, vector<vector<Rect> >& bounding_boxes);

void extractVocObjects(const string filename, vector<ObdObject>& objects, vector<VocObjectData>& object_data);

string getImagePath(const string& input_str);

void getClassImages_impl(const string& obj_class, const string& dataset_str, vector<ObdImage>& images, vector<char>& object_present);

void calcPrecRecall_impl(const vector<char>& ground_truth, const vector<float>& scores, vector<float>& precision, vector<float>& recall, float& ap, vector<size_t>& ranking, int recall_normalization = -1);

//test two bounding boxes to see if they meet the overlap criteria defined in the VOC documentation

float testBoundingBoxesForOverlap(const Rect detection, const Rect ground_truth);

//extract class and dataset name from a VOC-standard classification/detection results filename

void extractDataFromResultsFilename(const string& input_file, string& class_name, string& dataset_name);

//get classifier ground truth for a single image

bool getClassifierGroundTruthImage(const string& obj_class, const string& id);

//utility functions

void getSortOrder(const vector<float>& values, vector<size_t>& order, bool descending = true);

int stringToInteger(const string input_str);

void readFileToString(const string filename, string& file_contents);

string integerToString(const int input_int);

string checkFilenamePathsep(const string filename, bool add_trailing_slash = false);

void convertImageCodesToObdImages(const vector<string>& image_codes, vector<ObdImage>& images);

int extractXMLBlock(const string src, const string tag, const int searchpos, string& tag_contents);

//utility sorter

struct orderingSorter

{

bool operator ()(std::pair<size_t, vector<float>::const_iterator> const& a, std::pair<size_t, vector<float>::const_iterator> const& b)

{

return (*a.second) > (*b.second);

}

};

//data members

string m_vocPath;

string m_vocName;

//string m_resPath;

string m_annotation_path;

string m_image_path;

string m_imageset_path;

string m_class_imageset_path;

vector<string> m_classifier_gt_all_ids;

vector<char> m_classifier_gt_all_present;

string m_classifier_gt_class;

//data members

string m_train_set;

string m_test_set;

vector<string> m_object_classes;

float m_min_overlap;

bool m_sampled_ap;

{

string dataset_str;

//generate the filename of the classification ground-truth textfile for the object class

if (dataset == CV_OBD_TRAIN)

{

dataset_str = m_train_set;

} else {

dataset_str = m_test_set;

}

getClassImages_impl(obj_class, dataset_str, images, object_present);

{

//generate the filename of the classification ground-truth textfile for the object class

string gtFilename = m_class_imageset_path;

gtFilename.replace(gtFilename.find("%s"),2,obj_class);

gtFilename.replace(gtFilename.find("%s"),2,dataset_str);

//parse the ground truth file, storing in two separate vectors

//for the image code and the ground truth value

vector<string> image_codes;

readClassifierGroundTruth(gtFilename, image_codes, object_present);

//prepare output arrays

images.clear();

convertImageCodesToObdImages(image_codes, images);

{

vector<vector<VocObjectData> > object_data;

vector<VocGT> ground_truth;

getClassObjects(obj_class,dataset,images,objects,object_data,ground_truth);

{

//generate the filename of the classification ground-truth textfile for the object class

string gtFilename = m_class_imageset_path;

gtFilename.replace(gtFilename.find("%s"),2,obj_class);

if (dataset == CV_OBD_TRAIN)

{

gtFilename.replace(gtFilename.find("%s"),2,m_train_set);

} else {

gtFilename.replace(gtFilename.find("%s"),2,m_test_set);

}

//parse the ground truth file, storing in two separate vectors

//for the image code and the ground truth value

vector<string> image_codes;

vector<char> object_present;

readClassifierGroundTruth(gtFilename, image_codes, object_present);

//prepare output arrays

images.clear();

objects.clear();

object_data.clear();

ground_truth.clear();

string annotationFilename;

vector<ObdObject> image_objects;

vector<VocObjectData> image_object_data;

VocGT image_gt;

//transfer to output arrays and read in object data for each image

for (size_t i = 0; i < image_codes.size(); ++i)

{

ObdImage image = getObjects(obj_class, image_codes[i], image_objects, image_object_data, image_gt);

images.push_back(image);

objects.push_back(image_objects);

object_data.push_back(image_object_data);

ground_truth.push_back(image_gt);

}

{

vector<VocObjectData> object_data;

ObdImage image = getObjects(id, objects, object_data);

return image;

{

//first generate the filename of the annotation file

string annotationFilename = m_annotation_path;

annotationFilename.replace(annotationFilename.find("%s"),2,id);

//extract objects contained in the current image from the xml

extractVocObjects(annotationFilename,objects,object_data);

//generate image path from extracted string code

string path = getImagePath(id);

ObdImage image(id, path);

return image;

{

//extract object data (except for ground truth flag)

ObdImage image = getObjects(id,objects,object_data);

//pregenerate a flag to indicate whether the current class is present or not in the image

ground_truth = CV_VOC_GT_NONE;

//iterate through all objects in current image

for (size_t j = 0; j < objects.size(); ++j)

{

if (objects[j].object_class == obj_class)

{

if (object_data[j].difficult == false)

{

//if at least one non-difficult example is present, this flag is always set to CV_VOC_GT_PRESENT

ground_truth = CV_VOC_GT_PRESENT;

break;

} else {

//set if at least one object instance is present, but it is marked difficult

ground_truth = CV_VOC_GT_DIFFICULT;

}

}

}

return image;

{

vector<char>(images.size()).swap(ground_truth);

vector<ObdObject> objects;

vector<VocObjectData> object_data;

vector<char>::iterator gt_it = ground_truth.begin();

for (vector<ObdImage>::const_iterator it = images.begin(); it != images.end(); ++it, ++gt_it)

{

//getObjects(obj_class, it->id, objects, object_data, voc_ground_truth);

(*gt_it) = (getClassifierGroundTruthImage(obj_class, it->id));

}

{

vector<char>(images.size()).swap(ground_truth);

vector<ObdObject> objects;

vector<VocObjectData> object_data;

vector<char>::iterator gt_it = ground_truth.begin();

for (vector<string>::const_iterator it = images.begin(); it != images.end(); ++it, ++gt_it)

{

//getObjects(obj_class, (*it), objects, object_data, voc_ground_truth);

(*gt_it) = (getClassifierGroundTruthImage(obj_class, (*it)));

}

{

int recall_normalization = 0;

/* first create a list of indices referring to the elements of bounding_boxes and scores in

vector<ObdScoreIndexSorter> sorted_ids;

{

/* first count how many objects to allow preallocation */

size_t obj_count = 0;

CV_Assert(images.size() == bounding_boxes.size());

CV_Assert(scores.size() == bounding_boxes.size());

for (size_t im_idx = 0; im_idx < scores.size(); ++im_idx)

{

CV_Assert(scores[im_idx].size() == bounding_boxes[im_idx].size());

obj_count += scores[im_idx].size();

}

/* preallocate id vector */

sorted_ids.resize(obj_count);

/* now copy across scores and indexes to preallocated vector */

int flat_pos = 0;

for (size_t im_idx = 0; im_idx < scores.size(); ++im_idx)

{

for (size_t ob_idx = 0; ob_idx < scores[im_idx].size(); ++ob_idx)

{

sorted_ids[flat_pos].score = scores[im_idx][ob_idx];

sorted_ids[flat_pos].image_idx = (int)im_idx;

sorted_ids[flat_pos].obj_idx = (int)ob_idx;

++flat_pos;

}

}

/* and sort the vector in descending order of score */

std::sort(sorted_ids.begin(),sorted_ids.end());

std::reverse(sorted_ids.begin(),sorted_ids.end());

}

/* prepare ground truth + difficult vector (1st dimension) */

vector<vector<char> >(images.size()).swap(ground_truth);

vector<vector<char> >(images.size()).swap(detection_difficult);

vector<vector<char> > detected(images.size());

vector<vector<ObdObject> > img_objects(images.size());

vector<vector<VocObjectData> > img_object_data(images.size());

/* preload object ground truth bounding box data */

{

vector<vector<ObdObject> > img_objects_all(images.size());

vector<vector<VocObjectData> > img_object_data_all(images.size());

for (size_t image_idx = 0; image_idx < images.size(); ++image_idx)

{

/* prepopulate ground truth bounding boxes */

getObjects(images[image_idx].id, img_objects_all[image_idx], img_object_data_all[image_idx]);

/* meanwhile, also set length of target ground truth + difficult vector to same as number of object detections (2nd dimension) */

ground_truth[image_idx].resize(bounding_boxes[image_idx].size());

detection_difficult[image_idx].resize(bounding_boxes[image_idx].size());

}

/* save only instances of the object class concerned */

for (size_t image_idx = 0; image_idx < images.size(); ++image_idx)

{

for (size_t obj_idx = 0; obj_idx < img_objects_all[image_idx].size(); ++obj_idx)

{

if (img_objects_all[image_idx][obj_idx].object_class == obj_class)

{

img_objects[image_idx].push_back(img_objects_all[image_idx][obj_idx]);

img_object_data[image_idx].push_back(img_object_data_all[image_idx][obj_idx]);

}

}

detected[image_idx].resize(img_objects[image_idx].size(), false);

}

}

/* calculate the total number of objects in the ground truth for the current dataset */

{

vector<ObdImage> gt_images;

vector<char> gt_object_present;

getClassImages(obj_class, dataset, gt_images, gt_object_present);

for (size_t image_idx = 0; image_idx < gt_images.size(); ++image_idx)

{

vector<ObdObject> gt_img_objects;

vector<VocObjectData> gt_img_object_data;

getObjects(gt_images[image_idx].id, gt_img_objects, gt_img_object_data);

for (size_t obj_idx = 0; obj_idx < gt_img_objects.size(); ++obj_idx)

{

if (gt_img_objects[obj_idx].object_class == obj_class)

{

if ((gt_img_object_data[obj_idx].difficult == false) || (ignore_difficult == false))

++recall_normalization;

}

}

}

}

#ifdef PR_DEBUG

int printed_count = 0;

#endif

/* now iterate through detections in descending order of score, assigning to ground truth bounding boxes if possible */

for (size_t detect_idx = 0; detect_idx < sorted_ids.size(); ++detect_idx)

{

//read in indexes to make following code easier to read

int im_idx = sorted_ids[detect_idx].image_idx;

int ob_idx = sorted_ids[detect_idx].obj_idx;

//set ground truth for the current object to false by default

ground_truth[im_idx][ob_idx] = false;

detection_difficult[im_idx][ob_idx] = false;

float maxov = -1.0;

bool max_is_difficult = false;

int max_gt_obj_idx = -1;

//-- for each detected object iterate through objects present in the bounding box ground truth --

for (size_t gt_obj_idx = 0; gt_obj_idx < img_objects[im_idx].size(); ++gt_obj_idx)

{

if (detected[im_idx][gt_obj_idx] == false)

{

//check if the detected object and ground truth object overlap by a sufficient margin

float ov = testBoundingBoxesForOverlap(bounding_boxes[im_idx][ob_idx], img_objects[im_idx][gt_obj_idx].boundingBox);

if (ov != -1.0)

{

//if all conditions are met store the overlap score and index (as objects are assigned to the highest scoring match)

if (ov > maxov)

{

maxov = ov;

max_gt_obj_idx = (int)gt_obj_idx;

//store whether the maximum detection is marked as difficult or not

max_is_difficult = (img_object_data[im_idx][gt_obj_idx].difficult);

}

}

}

}

//-- if a match was found, set the ground truth of the current object to true --

if (maxov != -1.0)

{

CV_Assert(max_gt_obj_idx != -1);

ground_truth[im_idx][ob_idx] = true;

//store whether the maximum detection was marked as 'difficult' or not

detection_difficult[im_idx][ob_idx] = max_is_difficult;

//remove the ground truth object so it doesn't match with subsequent detected objects

//** this is the behaviour defined by the voc documentation **

detected[im_idx][max_gt_obj_idx] = true;

}

#ifdef PR_DEBUG

if (printed_count < 10)

{

cout << printed_count << ": id=" << images[im_idx].id << ", score=" << scores[im_idx][ob_idx] << " (" << ob_idx << ") [" << bounding_boxes[im_idx][ob_idx].x << "," <<

bounding_boxes[im_idx][ob_idx].y << "," << bounding_boxes[im_idx][ob_idx].width + bounding_boxes[im_idx][ob_idx].x <<

"," << bounding_boxes[im_idx][ob_idx].height + bounding_boxes[im_idx][ob_idx].y << "] detected=" << ground_truth[im_idx][ob_idx] <<

", difficult=" << detection_difficult[im_idx][ob_idx] << endl;

++printed_count;

/* print ground truth */

for (int gt_obj_idx = 0; gt_obj_idx < img_objects[im_idx].size(); ++gt_obj_idx)

{

cout << " GT: [" << img_objects[im_idx][gt_obj_idx].boundingBox.x << "," <<

img_objects[im_idx][gt_obj_idx].boundingBox.y << "," << img_objects[im_idx][gt_obj_idx].boundingBox.width + img_objects[im_idx][gt_obj_idx].boundingBox.x <<

"," << img_objects[im_idx][gt_obj_idx].boundingBox.height + img_objects[im_idx][gt_obj_idx].boundingBox.y << "]";

if (gt_obj_idx == max_gt_obj_idx) cout << " <--- (" << maxov << " overlap)";

cout << endl;

}

}

#endif

}

return recall_normalization;

{

CV_Assert(images.size() == scores.size());

string output_file_base, output_file;

if (dataset == CV_OBD_TRAIN)

{

output_file_base = out_dir + "/comp" + integerToString(competition) + "_cls_" + m_train_set + "_" + obj_class;

} else {

output_file_base = out_dir + "/comp" + integerToString(competition) + "_cls_" + m_test_set + "_" + obj_class;

}

output_file = output_file_base + ".txt";

//check if file exists, and if so create a numbered new file instead

if (overwrite_ifexists == false)

{

struct stat stFileInfo;

if (stat(output_file.c_str(),&stFileInfo) == 0)

{

string output_file_new;

int filenum = 0;

do

{

++filenum;

output_file_new = output_file_base + "_" + integerToString(filenum);

output_file = output_file_new + ".txt";

} while (stat(output_file.c_str(),&stFileInfo) == 0);

}

}

//output data to file

std::ofstream result_file(output_file.c_str());

if (result_file.is_open())

{

for (size_t i = 0; i < images.size(); ++i)

{

result_file << images[i].id << " " << scores[i] << endl;

}

result_file.close();

} else {

string err_msg = "could not open classifier results file '" + output_file + "' for writing. Before running for the first time, a 'results' subdirectory should be created within the VOC dataset base directory. e.g. if the VOC data is stored in /VOC/VOC2010 then the path /VOC/results must be created.";

CV_Error(CV_StsError,err_msg.c_str());

}

{

if ((competition < 1) && (competition != -1))

CV_Error(CV_StsBadArg,"competition argument should be a positive non-zero number or -1 to accept the default");

if ((number < 1) && (number != -1))

CV_Error(CV_StsBadArg,"number argument should be a positive non-zero number or -1 to accept the default");

string dset, task_type;

if (dataset == CV_OBD_TRAIN)

{

dset = m_train_set;

} else {

dset = m_test_set;

}

int comp = competition;

if (task == CV_VOC_TASK_CLASSIFICATION)

{

task_type = "cls";

if (comp == -1) comp = 1;

} else {

task_type = "det";

if (comp == -1) comp = 3;

}

stringstream ss;

if (number < 1)

{

ss << "comp" << comp << "_" << task_type << "_" << dset << "_" << obj_class << ".txt";

} else {

ss << "comp" << comp << "_" << task_type << "_" << dset << "_" << obj_class << "_" << number << ".txt";

}

string filename = ss.str();

return filename;

{

vector<char> res_ground_truth;

getClassifierGroundTruth(obj_class, images, res_ground_truth);

calcPrecRecall_impl(res_ground_truth, scores, precision, recall, ap, ranking);

{

vector<char> res_ground_truth;

getClassifierGroundTruth(obj_class, images, res_ground_truth);

vector<size_t> ranking;

calcPrecRecall_impl(res_ground_truth, scores, precision, recall, ap, ranking);

{

//read in classification results file

vector<string> res_image_codes;

vector<float> res_scores;

string input_file_std = checkFilenamePathsep(input_file);

readClassifierResultsFile(input_file_std, res_image_codes, res_scores);

//extract the object class and dataset from the results file filename

string class_name, dataset_name;

extractDataFromResultsFilename(input_file_std, class_name, dataset_name);

//generate the ground truth for the images extracted from the results file

vector<char> res_ground_truth;

getClassifierGroundTruth(class_name, res_image_codes, res_ground_truth);

if (outputRankingFile)

{

/* 1. store sorting order by score (descending) in 'order' */

vector<std::pair<size_t, vector<float>::const_iterator> > order(res_scores.size());

size_t n = 0;

for (vector<float>::const_iterator it = res_scores.begin(); it != res_scores.end(); ++it, ++n)

order[n] = make_pair(n, it);

std::sort(order.begin(),order.end(),orderingSorter());

/* 2. save ranking results to text file */

string input_file_std1 = checkFilenamePathsep(input_file);

size_t fnamestart = input_file_std1.rfind("/");

string scoregt_file_str = input_file_std1.substr(0,fnamestart+1) + "scoregt_" + class_name + ".txt";

std::ofstream scoregt_file(scoregt_file_str.c_str());

if (scoregt_file.is_open())

{

for (size_t i = 0; i < res_scores.size(); ++i)

{

scoregt_file << res_image_codes[order[i].first] << " " << res_scores[order[i].first] << " " << res_ground_truth[order[i].first] << endl;

}

scoregt_file.close();

} else {

string err_msg = "could not open scoregt file '" + scoregt_file_str + "' for writing.";

CV_Error(CV_StsError,err_msg.c_str());

}

}

//finally, calculate precision+recall+ap

vector<size_t> ranking;

calcPrecRecall_impl(res_ground_truth,res_scores,precision,recall,ap,ranking);

{

CV_Assert(ground_truth.size() == scores.size());

//add extra element for p-r at 0 recall (in case that first retrieved is positive)

vector<float>(scores.size()+1).swap(precision);

vector<float>(scores.size()+1).swap(recall);

// SORT RESULTS BY THEIR SCORE

/* 1. store sorting order in 'order' */

VocData::getSortOrder(scores, ranking);

#ifdef PR_DEBUG

std::ofstream scoregt_file("D:/pr.txt");

if (scoregt_file.is_open())

{

for (int i = 0; i < scores.size(); ++i)

{

scoregt_file << scores[ranking[i]] << " " << ground_truth[ranking[i]] << endl;

}

scoregt_file.close();

}

#endif

// CALCULATE PRECISION+RECALL

int retrieved_hits = 0;

int recall_norm;

if (recall_normalization != -1)

{

recall_norm = recall_normalization;

} else {

recall_norm = (int)std::count_if(ground_truth.begin(),ground_truth.end(),std::bind2nd(std::equal_to<char>(),(char)1));

}

ap = 0;

recall[0] = 0;

for (size_t idx = 0; idx < ground_truth.size(); ++idx)

{

if (ground_truth[ranking[idx]] != 0) ++retrieved_hits;

precision[idx+1] = static_cast<float>(retrieved_hits)/static_cast<float>(idx+1);

recall[idx+1] = static_cast<float>(retrieved_hits)/static_cast<float>(recall_norm);

if (idx == 0)

{

//add further point at 0 recall with the same precision value as the first computed point

precision[idx] = precision[idx+1];

}

if (recall[idx+1] == 1.0)

{

//if recall = 1, then end early as all positive images have been found

recall.resize(idx+2);

precision.resize(idx+2);

break;

}

}

/* ap calculation */

if (m_sampled_ap == false)

{

// FOR VOC2010+ AP IS CALCULATED FROM ALL DATAPOINTS

/* make precision monotonically decreasing for purposes of calculating ap */

vector<float> precision_monot(precision.size());

vector<float>::iterator prec_m_it = precision_monot.begin();

for (vector<float>::iterator prec_it = precision.begin(); prec_it != precision.end(); ++prec_it, ++prec_m_it)

{

vector<float>::iterator max_elem;

max_elem = std::max_element(prec_it,precision.end());

(*prec_m_it) = (*max_elem);

}

/* calculate ap */

for (size_t idx = 0; idx < (recall.size()-1); ++idx)

{

ap += (recall[idx+1] - recall[idx])*precision_monot[idx+1] + //no need to take min of prec - is monotonically decreasing

0.5f*(recall[idx+1] - recall[idx])*std::abs(precision_monot[idx+1] - precision_monot[idx]);

}

} else {

// FOR BEFORE VOC2010 AP IS CALCULATED BY SAMPLING PRECISION AT RECALL 0.0,0.1,..,1.0

for (float recall_pos = 0.f; recall_pos <= 1.f; recall_pos += 0.1f)

{

//find iterator of the precision corresponding to the first recall >= recall_pos

vector<float>::iterator recall_it = recall.begin();

vector<float>::iterator prec_it = precision.begin();

while ((*recall_it) < recall_pos)

{

++recall_it;

++prec_it;

if (recall_it == recall.end()) break;

}

/* if no recall >= recall_pos found, this level of recall is never reached so stop adding to ap */

if (recall_it == recall.end()) break;

/* if the prec_it is valid, compute the max precision at this level of recall or higher */

vector<float>::iterator max_prec = std::max_element(prec_it,precision.end());

ap += (*max_prec)/11;

}

}

{

CV_Assert(images.size() == scores.size());

// SORT RESULTS BY THEIR SCORE

/* 1. store sorting order in 'ranking' */

vector<size_t> ranking;

VocData::getSortOrder(scores, ranking);

// CALCULATE CONFUSION MATRIX ENTRIES

/* prepare object category headers */

output_headers = m_object_classes;

vector<float>(output_headers.size(),0.0).swap(output_values);

/* find the index of the target object class in the headers for later use */

int target_idx;

{

vector<string>::iterator target_idx_it = std::find(output_headers.begin(),output_headers.end(),obj_class);

/* if the target class can not be found, raise an exception */

if (target_idx_it == output_headers.end())

{

string err_msg = "could not find the target object class '" + obj_class + "' in list of valid classes.";

CV_Error(CV_StsError,err_msg.c_str());

}

/* convert iterator to index */

target_idx = (int)std::distance(output_headers.begin(),target_idx_it);

}

/* prepare variables related to calculating recall if using the recall threshold */

int retrieved_hits = 0;

int total_relevant = 0;