24 #include <fvclassifiers/surf.h>
30 #include <utils/time/clock.h>
31 #include <utils/time/tracker.h>
33 #include <surf/surflib.h>
41 #include <core/exception.h>
42 #include <core/exceptions/software.h>
43 #include <fvutils/color/colorspaces.h>
44 #include <fvutils/color/conversions.h>
45 #include <fvutils/readers/png.h>
46 #include <utils/logging/liblogger.h>
47 #include <utils/system/console_colors.h>
56 namespace firevision {
80 saveIpoints(std::string sFileName,
81 const std::vector<surf::Ipoint> &ipts,
86 std::cout <<
"Attempting to save interest points" << std::endl;
88 std::ofstream ipfile(sFileName.c_str());
90 std::cerr <<
"ERROR in loadIpoints(): "
91 <<
"Couldn't open file '" << sFileName.c_str() <<
"'!" << std::endl;
96 unsigned count = ipts.size();
100 ipfile << VLength + 1 << std::endl << count << std::endl;
102 ipfile << VLength << std::endl << count << std::endl;
113 for (
unsigned n = 0; n < ipts.size(); n++) {
115 sc = 2.5 * ipts[n].scale;
124 ipfile <<
" " << ipts[n].laplace;
127 for (
int i = 0; i < VLength; i++) {
128 ipfile <<
" " << ipts[n].ivec[i];
135 std::cout << count <<
" interest points found" << std::endl;
145 loadIpoints(std::string sFileName, std::vector<surf::Ipoint> &ipts,
bool bVerbose,
int &vlen_)
147 std::ifstream ipfile(sFileName.c_str());
150 std::cerr <<
"ERROR in loadIpoints(): "
151 <<
"Couldn't open file '" << sFileName.c_str() <<
"'!" << std::endl;
158 ipfile >> vlen_ >> count;
167 for (
unsigned n = 0; n < count; n++) {
170 ipfile >> x >> y >> a >> b >> c;
172 float det = sqrt((a - c) * (a - c) + 4.0 * b * b);
173 float e1 = 0.5 * (a + c + det);
174 float e2 = 0.5 * (a + c - det);
175 float l1 = (1.0 / sqrt(e1));
176 float l2 = (1.0 / sqrt(e2));
177 float sc = sqrt(l1 * l2);
181 ipts[n].scale = sc / 2.5;
182 ipfile >> ipts[n].laplace;
185 ipts[n].ivec =
new double[vlen_];
187 for (
int j = 0; j < vlen_; j++) {
188 ipfile >> ipts[n].ivec[j];
199 std::cout <<
"read in " << count <<
" interest points." << std::endl;
217 unsigned int min_match,
218 float min_match_ratio,
222 bool doubleImageSize,
229 obj_features_.clear();
230 obj_features_.reserve(1000);
232 min_match_ = min_match;
233 min_match_ratio_ = min_match_ratio;
235 samplingStep_ = samplingStep;
238 doubleImageSize_ = doubleImageSize;
239 initLobe_ = initLobe;
242 extended_ = extended;
243 indexSize_ = indexSize;
251 ttc_objconv_ = tt_->
add_class(
"ObjectConvert");
252 ttc_objfeat_ = tt_->
add_class(
"ObjectFeatures");
253 ttc_imgconv_ = tt_->
add_class(
"ImageConvert");
254 ttc_imgfeat_ = tt_->
add_class(
"ImageFeatures");
255 ttc_matchin_ = tt_->
add_class(
"Matching");
256 ttc_roimerg_ = tt_->
add_class(
"MergeROIs");
265 if ((dir = opendir(keypoints_dir.c_str())) == NULL) {
266 char *buffer =
new char[256];
267 sprintf(buffer,
"The directory %s does not exist!", keypoints_dir.c_str());
272 std::string object_file;
273 int num_obj_index = 0;
275 while ((ent = readdir(dir)) != NULL) {
276 if (strcmp(ent->d_name,
".") == 0 || strcmp(ent->d_name,
"..") == 0
277 || strcmp(ent->d_name,
".svn") == 0)
280 object_file = keypoints_dir + ent->d_name;
281 std::cout <<
"SurfClassifier: reading the following descriptor file" << object_file
284 obj_names_.push_back(object_file);
286 bool b_verbose = BVERBOSE;
287 loadIpoints(object_file, obj_features_[num_obj_index], b_verbose, vlen_);
294 num_obj_ = num_obj_index;
296 if (num_obj_index != 0) {
297 std::cout <<
"SurfClassifier: Reading successful" << std::endl;
303 std::cout <<
"SurfClassifier: The descriptor directory is probably empty since no objects were read off. Will instantiate a Surfclassifier with the png images directory") << std::endl;
333 unsigned int min_match,
334 float min_match_ratio,
338 bool doubleImageSize,
345 obj_features_.clear();
346 obj_features_.reserve(1000);
348 min_match_ = min_match;
349 min_match_ratio_ = min_match_ratio;
351 samplingStep_ = samplingStep;
354 doubleImageSize_ = doubleImageSize;
355 initLobe_ = initLobe;
358 extended_ = extended;
359 indexSize_ = indexSize;
367 ttc_objconv_ = tt_->
add_class(
"ObjectConvert");
368 ttc_objfeat_ = tt_->
add_class(
"ObjectFeatures");
369 ttc_imgconv_ = tt_->
add_class(
"ImageConvert");
370 ttc_imgfeat_ = tt_->
add_class(
"ImageFeatures");
371 ttc_matchin_ = tt_->
add_class(
"Matching");
372 ttc_roimerg_ = tt_->
add_class(
"MergeROIs");
381 std::string dir_path = object_dir;
383 if ((dir = opendir(dir_path.c_str())) == NULL) {
384 char *buffer =
new char[256];
385 sprintf(buffer,
"The directory %s does not exist!", dir_path.c_str());
391 std::string object_file;
392 int num_obj_index = 0;
394 while ((ent = readdir(dir)) != NULL) {
395 if (strcmp(ent->d_name,
".") == 0 || strcmp(ent->d_name,
"..") == 0
396 || strcmp(ent->d_name,
".svn") == 0)
399 object_file = dir_path + ent->d_name;
405 std::cout <<
"SurfClassifier(classify): opening object image file '" << object_file <<
"'"
415 surf::Image *simage_ =
new surf::Image(lwidth, lheight);
416 for (
unsigned int h = 0; h < lheight; ++h) {
417 for (
unsigned int w = 0; w < lwidth; ++w) {
418 simage_->setPix(w, h, (
double)buf[h * lwidth + w] / 255.f);
422 obj_img_ =
new surf::Image(simage_, doubleImageSize_);
445 std::vector<surf::Ipoint> obj_feature;
446 obj_features_.push_back(obj_feature);
447 obj_features_[num_obj_index].clear();
448 obj_features_[num_obj_index].reserve(1000);
449 obj_num_features_ = 0;
451 surf::FastHessian fh(obj_img_,
452 obj_features_[num_obj_index],
459 fh.getInterestPoints();
461 surf::Surf des(obj_img_,
468 vlen_ = des.getVectLength();
473 for (
unsigned n = 0; n < obj_features_[num_obj_index].size(); n++) {
475 des.setIpoint(&(obj_features_.at(num_obj_index).at(n)));
477 des.assignOrientation();
479 des.makeDescriptor();
482 obj_num_features_ = obj_features_[num_obj_index].size();
483 if (!obj_num_features_ > 0) {
486 std::cout <<
"SurfClassifier(classify): computed '" << obj_num_features_
487 <<
"' features from object" << std::endl;
490 sprintf(buffer,
"descriptors/%s-%d.surf", ent->d_name, num_obj_index);
491 std::string des_file_name = buffer;
493 bool b_verbose = BVERBOSE;
494 bool b_laplacian =
true;
496 obj_names_.push_back(des_file_name);
499 saveIpoints(des_file_name, obj_features_[num_obj_index], b_verbose, b_laplacian, vlen_);
511 num_obj_ = num_obj_index;
530 std::list<ROI> rv[num_obj_];
531 float match_ratios[num_obj_];
544 std::cout <<
"SurfClassifier(classify): copy imgdat to SURF Image" << std::endl;
562 for (
unsigned int h = 0; h <
_height; ++h) {
563 for (
unsigned int w = 0; w <
_width; ++w) {
564 simage_->setPix(w, h, (
double)
_src[h *
_width + w] / 255.f);
568 image_ =
new surf::Image(simage_, doubleImageSize_);
576 //surf::ImLoad::saveImage( "tst.pgm", simage_);
578 //surf::ImLoad::saveImage( "tst.pgm", image_);
580 PNMWriter pnm(PNM_PGM, "fvimg.pgm", _width, _height);
581 pnm.set_buffer(YUV422_PLANAR, _src );
584 PNGWriter pngw("fvimg.png", _width, _height);
585 pngw.set_buffer(YUV422_PLANAR, _src );
594 img_features_.clear();
595 img_features_.reserve(1000);
596 img_num_features_ = 0;
598 surf::FastHessian fh(image_,
606 std::cout <<
"surfclassifer/classify : getting interest points" << std::endl;
607 fh.getInterestPoints();
609 surf::Surf des(image_,
621 for (
unsigned n = 0; n < img_features_.size(); n++) {
624 des.setIpoint(&img_features_[n]);
626 des.assignOrientation();
628 des.makeDescriptor();
630 img_num_features_ = img_features_.size();
635 std::cout <<
"Extracted '" << img_num_features_ <<
"' image features" << std::endl;
640 std::cout <<
"SurfClassifier(classify): matching ..." << std::endl;
642 for (
unsigned j = 0; j < num_obj_; j++) {
643 std::vector<int> matches(obj_features_[j].size());
646 for (
unsigned i = 0; i < obj_features_[j].size(); i++) {
647 int match = findMatch((obj_features_[j])[i], img_features_);
652 ROI r((
int)(img_features_[matches[i]].x) - 5,
653 (
int)(img_features_[matches[i]].y) - 5,
668 std::cout <<
"SurfClassifier(classify) matched '" << c <<
fawkes::cnormal <<
"' of '"
669 << obj_features_[j].size() <<
"' features in scene. (for supplied object = " << j
673 <<
"' of '" << obj_features_[j].size()
674 <<
"' features in scene. (for supplied object = " << j << std::endl;
676 float match_ratio = ((float)c / (
float)obj_features_[j].size());
677 match_ratios[j] = match_ratio;
679 std::cout <<
"SurfClassifier(classify): match_ratio is '" << match_ratio
680 <<
"' and min_match_ratio is" << min_match_ratio_ << std::endl;
682 std::cout <<
"SurfClassifier(classify): computing ROI" << std::endl;
686 for (
unsigned i = 0; i < matches.size(); i++) {
687 if (matches[i] != -1) {
690 if ((
int)img_features_[matches[i]].x < x_min)
691 x_min = (
int)img_features_[matches[i]].x;
692 if ((
int)img_features_[matches[i]].y < y_min)
693 y_min = (
int)img_features_[matches[i]].y;
694 if ((
int)img_features_[matches[i]].x > x_max)
695 x_max = (
int)img_features_[matches[i]].x;
696 if ((
int)img_features_[matches[i]].y > y_max)
697 y_max = (
int)img_features_[matches[i]].y;
700 if ((c != 0) && ((unsigned)c > min_match_) && (match_ratio > min_match_ratio_)
701 && (x_max - x_min != 0) && (y_max - y_min != 0)) {
702 std::cout <<
"SurfClassifier(classify): c='" << c <<
"' min_match_='" << min_match_ <<
"'."
709 std::cout <<
" clearing ROI-list (no or too few matches or [0,0]-roi!)" << std::endl;
731 float min_ratio_tmp = -1.0;
732 int min_ratio_index = -1;
733 for (
unsigned int i = 0; i < num_obj_; i++) {
734 if (match_ratios[i] > min_ratio_tmp) {
735 min_ratio_tmp = match_ratios[i];
740 std::list<ROI> *final_rv =
new std::list<ROI>;
742 final_rv->assign(rv[min_ratio_index].begin(), rv[min_ratio_index].end());
744 std::string first_not(
".-");
745 int first_not_index = obj_names_[min_ratio_index].find_first_of(first_not);
746 std::string obj_name_tmp(obj_names_[min_ratio_index]);
747 obj_name_tmp.erase(first_not_index);
749 std::cout <<
"SurfClassifier(classify): done, ... returning '" << rv->size()
750 <<
"' ROIs. The object class is " << min_ratio_index <<
"and object name is "
756 SurfClassifier::findMatch(
const surf::Ipoint &ip1,
const std::vector<surf::Ipoint> &ipts)
758 double mind = 1e100, second = 1e100;
763 for (
unsigned i = 0; i < ipts.size(); i++) {
765 if (ipts[i].laplace != ip1.laplace)
768 double d = distSquare(ipts[i].ivec, ip1.ivec, vlen_);
774 }
else if (d < second) {
779 if (mind < 0.5 * second)
786 SurfClassifier::distSquare(
double *v1,
double *v2,
int n)
792 dsq += (*v1 - *v2) * (*v1 - *v2);
Base class for exceptions in Fawkes.
static void log_error(const char *component, const char *format,...)
Log error message.
unsigned int add_class(std::string name)
Add a new class.
void ping_end(unsigned int cls)
End of given class task.
void ping_start(unsigned int cls)
Start of given class task.
Classifier to extract regions of interest.
unsigned int _height
Height in pixels of _src buffer.
unsigned char * _src
Source buffer, encoded as YUV422_PLANAR.
unsigned int _width
Width in pixels of _src buffer.
virtual void set_buffer(unsigned char *yuv422planar_buffer)
Set buffer that the read image should be written to.
virtual unsigned int pixel_height()
Get height of read image in pixels.
virtual colorspace_t colorspace()
Get colorspace from the just read image.
virtual void read()
Read data from file.
virtual unsigned int pixel_width()
Get width of read image in pixels.
unsigned int num_hint_points
Minimum estimate of points in ROI that are attributed to the ROI hint.
SurfClassifier(std::string keypoints_descriptor_txt_file, unsigned int min_match=5, float min_match_ratio=MIN_MATCH_RATIO, int samplingStep=2, int octaves=4, double thres=4.0, bool doubleImageSize=false, int initLobe=3, bool upright=false, bool extended=false, int indexSize=4)
Constructor.
virtual ~SurfClassifier()
Destructor.
virtual std::list< ROI > * classify()
Classify image.
static std::string cnormal
Print normal on console, without colors, depends on console settings.
static std::string cblue
Print blue on console.
static std::string cgreen
Print green on console.