// Copyright (C) 2013 Davis E. King (davis@dlib.net)
// License: Boost Software License See LICENSE.txt for the full license.
#undef DLIB_SCAN_IMAGE_bOXES_ABSTRACT_Hh_
#ifdef DLIB_SCAN_IMAGE_bOXES_ABSTRACT_Hh_
#include "../matrix.h"
#include "../geometry.h"
#include "../image_processing.h"
#include "../array2d.h"
#include "full_object_detection_abstract.h"
#include "../image_transforms/segment_image_abstract.h"
#include <vector>
namespace dlib
{
// ----------------------------------------------------------------------------------------
class default_box_generator
{
/*!
WHAT THIS OBJECT REPRESENTS
This is a function object that takes in an image and outputs a set of
candidate object locations. It is also the default box generator used by
the scan_image_boxes object defined below.
!*/
public:
template <typename image_type>
void operator() (
const image_type& img,
std::vector<rectangle>& rects
) const
/*!
ensures
- #rects == the set of candidate object locations which should be searched
inside img. That is, these are the rectangles which might contain
objects of interest within the given image.
!*/
{
rects.clear();
find_candidate_object_locations(img, rects);
}
};
inline void serialize (const default_box_generator&, std::ostream& ) {}
inline void deserialize( default_box_generator&, std::istream& ) {}
/*!
ensures
- provides serialization support.
!*/
// ----------------------------------------------------------------------------------------
template <
typename Feature_extractor_type,
typename Box_generator = default_box_generator
>
class scan_image_boxes : noncopyable
{
/*!
REQUIREMENTS ON Feature_extractor_type
- must be an object with an interface compatible with the hashed_feature_image
object defined in dlib/image_keypoint/hashed_feature_image_abstract.h or
with the nearest_neighbor_feature_image object defined in
dlib/image_keypoint/nearest_neighbor_feature_image_abstract.h
REQUIREMENTS ON Box_generator
- must be an object with an interface compatible with the
default_box_generator object defined at the top of this file.
INITIAL VALUE
- get_num_spatial_pyramid_levels() == 3
- is_loaded_with_image() == false
WHAT THIS OBJECT REPRESENTS
This object is a tool for running a classifier over an image with the goal
of localizing each object present. The localization is in the form of the
bounding box around each object of interest.
Unlike the scan_image_pyramid object which scans a fixed sized window over
an image pyramid, the scan_image_boxes tool allows you to define your own
list of "candidate object locations" which should be evaluated. This is
simply a list of rectangle objects which might contain objects of interest.
The scan_image_boxes object will then evaluate the classifier at each of
these locations and return the subset of rectangles which appear to have
objects in them. The candidate object location generation is provided by
the Box_generator that is passed in as a template argument.
This object can also be understood as a general tool for implementing the
spatial pyramid models described in the paper:
Beyond Bags of Features: Spatial Pyramid Matching for Recognizing
Natural Scene Categories by Svetlana Lazebnik, Cordelia Schmid,
and Jean Ponce
The classifiers used by this object have three parts:
1. The underlying feature extraction provided by Feature_extractor_type
objects, which associate a vector with each location in an image.
2. A rule for extracting a feature vector from a candidate object
location. In this object we use the spatial pyramid matching method.
This means we cut an object's detection window into a set of "feature
extraction regions" and extract a bag-of-words vector from each
before finally concatenating them to form the final feature vector
representing the entire object window. The set of feature extraction
regions can be configured by the user by calling
set_num_spatial_pyramid_levels(). To be a little more precise, the
feature vector for a candidate object window is defined as follows:
- Let N denote the number of feature extraction zones.
- Let M denote the dimensionality of the vectors output by
Feature_extractor_type objects.
- Let F(i) == the M dimensional vector which is the sum of all
vectors given by our Feature_extractor_type object inside the
i-th feature extraction zone. So this is notionally a
bag-of-words vector from the i-th zone.
- Then the feature vector for an object window is an M*N
dimensional vector [F(1) F(2) F(3) ... F(N)] (i.e. it is a
concatenation of the N vectors). This feature vector can be
thought of as a collection of N bags-of-words, each bag coming
from a spatial location determined by one of the feature
extraction zones.
3. A weight vector and a threshold value. The dot product between the
weight vector and the feature vector for a candidate object location
gives the score of the location. If this score is greater than the
threshold value then the candidate object location is output as a
detection.
THREAD SAFETY
Concurrent access to an instance of this object is not safe and should be
protected by a mutex lock except for the case where you are copying the
configuration (via copy_configuration()) of a scan_image_boxes object to
many other threads. In this case, it is safe to copy the configuration of
a shared object so long as no other operations are performed on it.
!*/
public:
typedef matrix<double,0,1> feature_vector_type;
typedef Feature_extractor_type feature_extractor_type;
typedef Box_generator box_generator;
scan_image_boxes (
);
/*!
ensures
- this object is properly initialized
!*/
template <
typename image_type
>
void load (
const image_type& img
);
/*!
requires
- image_type must be a type with the following properties:
- image_type objects can be loaded into Feature_extractor_type
objects via Feature_extractor_type::load().
- image_type objects can be passed to the first argument of
Box_generator::operator()
ensures
- #is_loaded_with_image() == true
- This object is ready to run a classifier over img to detect object
locations. Call detect() to do this.
!*/
bool is_loaded_with_image (
) const;
/*!
ensures
- returns true if this object has been loaded with an image to process and
false otherwise.
!*/
const feature_extractor_type& get_feature_extractor (
) const;
/*!
ensures
- returns a const reference to the feature_extractor_type object used
internally for local feature extraction.
!*/
void copy_configuration(
const feature_extractor_type& fe
);
/*!
ensures
- This function performs the equivalent of
get_feature_extractor().copy_configuration(fe) (i.e. this function allows
you to configure the parameters of the underlying feature extractor used
by a scan_image_boxes object)
!*/
void copy_configuration(
const box_generator& bg
);
/*!
ensures
- #get_box_generator() == bg
(i.e. this function allows you to configure the parameters of the
underlying box generator used by a scan_image_boxes object)
!*/
const box_generator& get_box_generator (
) const;
/*!
ensures
- returns the box_generator used by this object to generate candidate
object locations.
!*/
void copy_configuration (
const scan_image_boxes& item
);
/*!
ensures
- Copies all the state information of item into *this, except for state
information populated by load(). More precisely, given two scan_image_boxes
objects S1 and S2, the following sequence of instructions should always
result in both of them having the exact same state:
S2.copy_configuration(S1);
S1.load(img);
S2.load(img);
!*/
long get_num_dimensions (
) const;
/*!
ensures
- returns the number of dimensions in the feature vector for a candidate
object location. This value is the dimensionality of the underlying
feature vectors produced by Feature_extractor_type times the number of
feature extraction regions used. Note that the number of feature
extraction regions used is a function of
get_num_spatial_pyramid_levels().
!*/
unsigned long get_num_spatial_pyramid_levels (
) const;
/*!
ensures
- returns the number of layers in the spatial pyramid. For example, if
this function returns 1 then it means we use a simple bag-of-words
representation over the whole object window. If it returns 2 then it
means the feature representation is the concatenation of 5 bag-of-words
vectors, one from the entire object window and 4 others from 4 different
parts of the object window. If it returns 3 then there are 1+4+16
bag-of-words vectors concatenated together in the feature representation,
and so on.
!*/
void set_num_spatial_pyramid_levels (
unsigned long levels
);
/*!
requires
- levels > 0
ensures
- #get_num_spatial_pyramid_levels() == levels
!*/
void detect (
const feature_vector_type& w,
std::vector<std::pair<double, rectangle> >& dets,
const double thresh
) const;
/*!
requires
- w.size() >= get_num_dimensions()
- is_loaded_with_image() == true
ensures
- Scans over all the candidate object locations as discussed in the WHAT
THIS OBJECT REPRESENTS section and stores all detections into #dets.
- for all valid i:
- #dets[i].second == The candidate object location which produced this
detection. This rectangle gives the location of the detection.
- #dets[i].first == The score for this detection. This value is equal
to dot(w, feature vector for this candidate object location).
- #dets[i].first >= thresh
- #dets will be sorted in descending order.
(i.e. #dets[i].first >= #dets[j].first for all i, and j>i)
- Elements of w beyond index get_num_dimensions()-1 are ignored. I.e. only
the first get_num_dimensions() are used.
- Note that no form of non-max suppression is performed. If a locations
has a score >= thresh then it is reported in #dets.
!*/
void get_feature_vector (
const full_object_detection& obj,
feature_vector_type& psi
) const;
/*!
requires
- obj.num_parts() == 0
- is_loaded_with_image() == true
- psi.size() >= get_num_dimensions()
(i.e. psi must have preallocated its memory before this function is called)
ensures
- This function allows you to determine the feature vector used for a
candidate object location output from detect(). Note that this vector is
added to psi. Note also that you must use get_full_object_detection() to
convert a rectangle from detect() into the needed full_object_detection.
- The dimensionality of the vector added to psi is get_num_dimensions(). This
means that elements of psi after psi(get_num_dimensions()-1) are not modified.
- Since scan_image_boxes only searches a limited set of object locations,
not all possible rectangles can be output by detect(). So in the case
where obj.get_rect() could not arise from a call to detect(), this
function will map obj.get_rect() to the nearest possible rectangle and
then add the feature vector for the mapped rectangle into #psi.
- get_best_matching_rect(obj.get_rect()) == the rectangle obj.get_rect()
gets mapped to for feature extraction.
!*/
full_object_detection get_full_object_detection (
const rectangle& rect,
const feature_vector_type& w
) const;
/*!
ensures
- returns full_object_detection(rect)
(This function is here only for compatibility with the scan_image_pyramid
object)
!*/
const rectangle get_best_matching_rect (
const rectangle& rect
) const;
/*!
requires
- is_loaded_with_image() == true
ensures
- Since scan_image_boxes only searches a limited set of object locations,
not all possible rectangles can be represented. Therefore, this function
allows you to supply a rectangle and obtain the nearest possible
candidate object location rectangle.
!*/
unsigned long get_num_detection_templates (
) const { return 1; }
/*!
ensures
- returns 1. Note that this function is here only for compatibility with
the scan_image_pyramid object. Notionally, its return value indicates
that a scan_image_boxes object is always ready to detect objects once
an image has been loaded.
!*/
unsigned long get_num_movable_components_per_detection_template (
) const { return 0; }
/*!
ensures
- returns 0. Note that this function is here only for compatibility with
the scan_image_pyramid object. Its return value means that this object
does not support using movable part models.
!*/
};
// ----------------------------------------------------------------------------------------
template <
typename Feature_extractor_type,
typename Box_generator
>
void serialize (
const scan_image_boxes<Feature_extractor_type,Box_generator>& item,
std::ostream& out
);
/*!
provides serialization support
!*/
template <
typename Feature_extractor_type,
typename Box_generator
>
void deserialize (
scan_image_boxes<Feature_extractor_type,Box_generator>& item,
std::istream& in
);
/*!
provides deserialization support
!*/
// ----------------------------------------------------------------------------------------
}
#endif // DLIB_SCAN_IMAGE_bOXES_ABSTRACT_Hh_