// The contents of this file are in the public domain. See LICENSE_FOR_EXAMPLE_PROGRAMS.txt
/*
This is an example illustrating the use of dlib's bag-of-visual-word based
tools for detecting objects in images. In this example we will create three
simple images, each containing some white squares. We will then use the
sliding window classifier tools to learn to detect these squares.
If the objects you want to detect are somewhat rigid in appearance (e.g.
faces, pedestrians, etc.) then you should try the methods shown in the
fhog_object_detector_ex.cpp example program before trying to use the
bag-of-visual-word tools shown in this example.
*/
#include <dlib/svm_threaded.h>
#include <dlib/gui_widgets.h>
#include <dlib/array.h>
#include <dlib/array2d.h>
#include <dlib/image_keypoint.h>
#include <dlib/image_processing.h>
#include <iostream>
#include <fstream>
using namespace std;
using namespace dlib;
// ----------------------------------------------------------------------------------------
template <
typename image_array_type
>
void make_simple_test_data (
image_array_type& images,
std::vector<std::vector<rectangle> >& object_locations
)
/*!
ensures
- #images.size() == 3
- #object_locations.size() == 3
- Creates some simple images to test the object detection routines. In particular,
this function creates images with white 70x70 squares in them. It also stores
the locations of these squares in object_locations.
- for all valid i:
- object_locations[i] == A list of all the white rectangles present in images[i].
!*/
{
images.clear();
object_locations.clear();
images.resize(3);
images[0].set_size(400,400);
images[1].set_size(400,400);
images[2].set_size(400,400);
// set all the pixel values to black
assign_all_pixels(images[0], 0);
assign_all_pixels(images[1], 0);
assign_all_pixels(images[2], 0);
// Now make some squares and draw them onto our black images. All the
// squares will be 70 pixels wide and tall.
std::vector<rectangle> temp;
temp.push_back(centered_rect(point(100,100), 70,70));
fill_rect(images[0],temp.back(),255); // Paint the square white
temp.push_back(centered_rect(point(200,300), 70,70));
fill_rect(images[0],temp.back(),255); // Paint the square white
object_locations.push_back(temp);
temp.clear();
temp.push_back(centered_rect(point(140,200), 70,70));
fill_rect(images[1],temp.back(),255); // Paint the square white
temp.push_back(centered_rect(point(303,200), 70,70));
fill_rect(images[1],temp.back(),255); // Paint the square white
object_locations.push_back(temp);
temp.clear();
temp.push_back(centered_rect(point(123,121), 70,70));
fill_rect(images[2],temp.back(),255); // Paint the square white
object_locations.push_back(temp);
// corrupt each image with random noise just to make this a little more
// challenging
dlib::rand rnd;
for (unsigned long i = 0; i < images.size(); ++i)
{
for (long r = 0; r < images[i].nr(); ++r)
{
for (long c = 0; c < images[i].nc(); ++c)
{
images[i][r][c] = put_in_range(0,255,images[i][r][c] + 40*rnd.get_random_gaussian());
}
}
}
}
// ----------------------------------------------------------------------------------------
int main()
{
try
{
// The first thing we do is create the set of 3 images discussed above.
dlib::array<array2d<unsigned char> > images;
std::vector<std::vector<rectangle> > object_locations;
make_simple_test_data(images, object_locations);
/*
This next block of code specifies the type of sliding window classifier we will
be using to detect the white squares. The most important thing here is the
scan_image_pyramid template. Instances of this template represent the core
of a sliding window classifier. To go into more detail, the sliding window
classifiers used by this object have three parts:
1. The underlying feature extraction. See the dlib documentation for a detailed
discussion of how the hashed_feature_image and hog_image feature extractors
work. However, to understand this example, all you need to know is that the
feature extractor associates a vector with each location in an image. This
vector is supposed to capture information which describes how parts of the
image look. Importantly, it should do this in a way that is relevant to the
problem you are trying to solve.
2. A detection template. This is a rectangle which defines the shape of a
sliding window (i.e. the object_box), as well as a set of rectangular feature
extraction regions inside it. This set of regions defines the spatial
structure of the overall feature extraction within a sliding window. In
particular, each location of a sliding window has a feature vector
associated with it. This feature vector is defined as follows:
- Let N denote the number of feature extraction zones.
- Let M denote the dimensionality of the vectors output by Feature_extractor_type
objects.
- Let F(i) == the M dimensional vector which is the sum of all vectors
given by our Feature_extractor_type object inside the ith feature extraction
zone.
- Then the feature vector for a sliding window is an M*N dimensional vector
[F(1) F(2) F(3) ... F(N)] (i.e. it is a concatenation of the N vectors).
This feature vector can be thought of as a collection of N "bags of features",
each bag coming from a spatial location determined by one of the rectangular
feature extraction zones.
3. A weight vector and a threshold value. The dot product between the weight
vector and the feature vector for a sliding window location gives the score
of the window. If this score is greater than the threshold value then the
window location is output as a detection. You don't need to determine these
parameters yourself. They are automatically populated by the
structural_object_detection_trainer.
The sliding window classifiers described above are applied to every level of an
image pyramid. So you need to tell scan_image_pyramid what kind of pyramid you want
to use. In this case we are using pyramid_down<2> which downsamples each pyramid
layer by half (if you want to use a finer image pyramid then just change the
template argument to a larger value. For example, using pyramid_down<5> would
downsample each layer by a ratio of 5 to 4).
Finally, some of the feature extraction zones are allowed to move freely within the
object box. This means that when we are sliding the classifier over an image, some
feature extraction zones are stationary (i.e. always in the same place relative to
the object box) while others are allowed to move anywhere within the object box. In
particular, the movable regions are placed at the locations that maximize the score
of the classifier. Note further that each of the movable feature extraction zones
must pass a threshold test for it to be included. That is, if the score that a
movable zone would contribute to the overall score for a sliding window location is
not positive then that zone is not included in the feature vector (i.e. its part of
the feature vector is set to zero. This way the length of the feature vector stays
constant). This movable region construction allows us to represent objects with
parts that move around relative to the object box. For example, a human has hands
but they aren't always in the same place relative to a person's bounding box.
However, to keep this example program simple, we will only be using stationary
feature extraction regions.
*/
typedef hashed_feature_image<hog_image<3,3,1,4,hog_signed_gradient,hog_full_interpolation> > feature_extractor_type;
typedef scan_image_pyramid<pyramid_down<2>, feature_extractor_type> image_scanner_type;
image_scanner_type scanner;
// The hashed_feature_image in the scanner needs to be supplied with a hash function capable
// of hashing the outputs of the hog_image. Calling this function will set it up for us. The
// 10 here indicates that it will hash HOG vectors into the range [0, pow(2,10)). Therefore,
// the feature vectors output by the hashed_feature_image will have dimension pow(2,10).
setup_hashed_features(scanner, images, 10);
// We should also tell the scanner to use the uniform feature weighting scheme
// since it works best on the data in this example. If you don't call this
// function then it will use a slightly different weighting scheme which can give
// improved results on many normal image types.
use_uniform_feature_weights(scanner);
// We also need to setup the detection templates the scanner will use. It is important that
// we add detection templates which are capable of matching all the output boxes we want to learn.
// For example, if object_locations contained a rectangle with a height to width ratio of 10 but
// we only added square detection templates then it would be impossible to detect this non-square
// rectangle. The setup_grid_detection_templates_verbose() routine will take care of this for us by
// looking at the contents of object_locations and automatically picking an appropriate set. Also,
// the final arguments indicate that we want our detection templates to have 4 feature extraction
// regions laid out in a 2x2 regular grid inside each sliding window.
setup_grid_detection_templates_verbose(scanner, object_locations, 2, 2);
// Now that we have defined the kind of sliding window classifier system we want and stored
// the details into the scanner object we are ready to use the structural_object_detection_trainer
// to learn the weight vector and threshold needed to produce a complete object detector.
structural_object_detection_trainer<image_scanner_type> trainer(scanner);
trainer.set_num_threads(4); // Set this to the number of processing cores on your machine.
// There are a variety of other useful parameters to the structural_object_detection_trainer.
// Examples of the ones you are most likely to use follow (see dlib documentation for what they do):
//trainer.set_match_eps(0.80);
//trainer.set_c(1.0);
//trainer.set_loss_per_missed_target(1);
//trainer.set_loss_per_false_alarm(1);
// Do the actual training and save the results into the detector object.
object_detector<image_scanner_type> detector = trainer.train(images, object_locations);
// We can easily test the new detector against our training data. This print statement will indicate that it
// has perfect precision and recall on this simple task. It will also print the average precision (AP).
cout << "Test detector (precision,recall,AP): " << test_object_detection_function(detector, images, object_locations) << endl;
// The cross validation should also indicate perfect precision and recall.
cout << "3-fold cross validation (precision,recall,AP): "
<< cross_validate_object_detection_trainer(trainer, images, object_locations, 3) << endl;
// Let's display the output of the detector along with our training images.
image_window win;
for (unsigned long i = 0; i < images.size(); ++i)
{
// Run the detector on images[i]
const std::vector<rectangle> rects = detector(images[i]);
cout << "Number of detections: "<< rects.size() << endl;
// Put the image and detections into the window.
win.clear_overlay();
win.set_image(images[i]);
win.add_overlay(rects, rgb_pixel(255,0,0));
cout << "Hit enter to see the next image.";
cin.get();
}
// Finally, note that the detector can be serialized to disk just like other dlib objects.
serialize("object_detector.dat") << detector;
// Recall from disk.
deserialize("object_detector.dat") >> detector;
}
catch (exception& e)
{
cout << "\nexception thrown!" << endl;
cout << e.what() << endl;
}
}
// ----------------------------------------------------------------------------------------