update

Current Status
==============

1. complete the training part of JDA
2. use three scaled image for feature extraction according to #6
3. test the training with small model K=5, pos size = 3000, background image size = 9000

TODO
====

0. fix bugs of training part
1. impl the test part of JDA
2. impl the detect part of JDA
3. may use relateive delta shape in global regression, currently use absolute shape
4. data augmentation during the hard negative mining due to lacking of negative samples

Author: luoyetx

.gitignore

@@ -4,3 +4,4 @@ build/
 *.zip
 *.jpg
 *.png
+*.JPEG

CMakeLists.txt

@@ -7,5 +7,5 @@ include(libjda.cmake)
 
 file(GLOB SRC src/*.cpp)
 
-add_executable(${PROJECT_NAME} ${SRC})
-target_link_libraries(${PROJECT_NAME} libjda ${OpenCV_LIBS})
+add_executable(jda ${SRC})
+target_link_libraries(jda libjda ${OpenCV_LIBS})

include/jda/cart.hpp

@@ -26,16 +26,19 @@ class Cart {
 public:
     Cart();
     ~Cart();
-    Cart(const Cart& other);
-    Cart& operator=(const Cart& other);
+    //Cart(const Cart& other);
+    //Cart& operator=(const Cart& other);
     /**
      * Initialize Cart
+     * :input stage:            which stage this cart lie in
+     * :input landmark_id:      which landmark this cart training for regression
      */
     void Initialize(int stage, int landmark_id);
 
 public:
     /**
-     * Generate feature pool
+     * Generate feature pool, the pool size is determined by Config.feats[stage]
+     * :output feature_pool:    feature pool
      */
     void GenFeaturePool(std::vector<Feature>& feature_pool);
     /**
@@ -44,35 +47,50 @@ class Cart {
     void Train(DataSet& pos, DataSet& neg);
     /**
      * Split node with training data
+     * :input pos:          positive dataset
+     * :input neg:          negative dataset
+     * :input pos_idx:      index of used positive dataset
+     * :input neg_idx:      index of used negative dataset
+     * :input node_idx:     index of current node in this cart
      */
     void SplitNode(DataSet& pos, std::vector<int>& pos_idx, \
                    DataSet& neg, std::vector<int>& neg_idx, \
                    int node_idx);
     /**
      * Classification
+     * :input pos_feature:  pos feature
+     * :input neg_feature:  neg feature
+     * :output feature_id:  which feature we should use
+     * :output threshold:   split threshold
      *
      * split node with classification, minimize binary entropy of pos and neg
      * `f = argmax_{f \in F} H_{root} - (H_{left} + H_{right})`
      */
-    void SplitNodeWithClassification(cv::Mat_<int>& pos_feature, \
-                                     cv::Mat_<int>& neg_feature, \
-                                     int& feature_id, int& threshold);
+    static void SplitNodeWithClassification(const cv::Mat_<int>& pos_feature, \
+                                            const cv::Mat_<int>& neg_feature, \
+                                            int& feature_id, int& threshold);
     /**
      * Regression
+     * :input pos_feature:  pos feature
+     * :input neg_feature:  neg feature
+     * :output feature_id:  which feature we should use
+     * :output threshold:   split threshold
      *
      * split node with regression, minimize variance of shape_residual
      * `f = argmax_{f \in F} S_{root} - (S_{left} + S_{right})`
      */
-    void SplitNodeWithRegression(cv::Mat_<int>& pos_feature, \
-                                 cv::Mat_<double>& shape_residual, \
-                                 int& feature_id, int& threshold);
+    static void SplitNodeWithRegression(const cv::Mat_<int>& pos_feature, \
+                                        const cv::Mat_<double>& shape_residual, \
+                                        int& feature_id, int& threshold);
 
 public:
     /**
      * Forward a data point to leaf node
-     * :return:     leaf node index in this tree, start from 0
+     * :input img:      region
+     * :input shape:    shape
+     * :return:         leaf node index in this tree, start from 0
      */
-    int Forward(const cv::Mat& img, const cv::Mat_<double>& shape);
+    int Forward(const cv::Mat& img, const cv::Mat_<double>& shape) const;
 
 public:
     int stage; // cascade stage
@@ -100,8 +118,8 @@ class BoostCart {
 public:
     BoostCart();
     ~BoostCart();
-    BoostCart(const BoostCart& other);
-    BoostCart& operator=(const BoostCart& other);
+    //BoostCart(const BoostCart& other);
+    //BoostCart& operator=(const BoostCart& other);
     void Initialize(int stage);
 
 public:
@@ -115,7 +133,7 @@ class BoostCart {
      * we only use DataSet of pos, X = lbf, Y = shape_residual
      * see more detail on paper in section 4
      */
-    void GlobalRegression(const cv::Mat_<int>& lbf, \
+    void GlobalRegression(const std::vector<cv::Mat_<int> >& lbf, \
                           const cv::Mat_<double>& shape_residual);
     /**
      * Set Join Cascador
@@ -125,20 +143,21 @@ class BoostCart {
 public:
     /**
      * Generate Local Binary Feature
+     * :input img:      region
+     * :input shape:    shape
      * :return:         one row of local binary feature
      */
-    cv::Mat_<int> GenLBF(const cv::Mat& img, const cv::Mat_<double>& shape);
+    cv::Mat_<int> GenLBF(const cv::Mat& img, const cv::Mat_<double>& shape) const;
     /**
      * Generate delta shape with given lbf
+     * :input lbf:      lbf generated by `GenLBF`
      * :return:         one row of delta shape
      */
-    cv::Mat_<double> GenDeltaShape(const cv::Mat_<int>& lbf);
+    cv::Mat_<double> GenDeltaShape(const cv::Mat_<int>& lbf) const;
 
 public:
     int K; // number of carts
     int stage; // which stage this boost cart lies
-    double tp_rate; // true positive rate
-    double fn_rate; // false negative rate
 
     std::vector<Cart> carts; // boosted carts
     cv::Mat_<double> w; // weight of global regression

include/jda/cascador.hpp

@@ -15,8 +15,8 @@ class JoinCascador {
 public:
     JoinCascador();
     ~JoinCascador();
-    JoinCascador(const JoinCascador& other);
-    JoinCascador& operator=(const JoinCascador& other);
+    //JoinCascador(const JoinCascador& other);
+    //JoinCascador& operator=(const JoinCascador& other);
     void Initialize(int T);
 
 public:
@@ -26,6 +26,10 @@ class JoinCascador {
      * See Full Algorithm on paper `Algorithm 3`
      */
     void Train(DataSet& pos, DataSet& neg);
+    /**
+     * Snapshot current model
+     */
+    void Snapshot();
     /**
      * Write parameters to a binary file
      */
@@ -48,7 +52,7 @@ class JoinCascador {
      * a face score for this region. The training status is based on `current_stage_idx`
      * and `current_cart_idx`.
      */
-    bool Validate(cv::Mat& region, double& score, cv::Mat_<double>& shape);
+    bool Validate(const cv::Mat& region, double& score, cv::Mat_<double>& shape) const;
 
 public:
     int T; // number of stages

include/jda/common.hpp

@@ -1,6 +1,7 @@
 #ifndef COMMON_HPP_
 #define COMMON_HPP_
 
+#include <cassert>
 #include <opencv2/core/core.hpp>
 
 /**
@@ -9,31 +10,62 @@
  * usage:
  *      TIMER_BEGIN
  *          ....
- *          TIMER_NOW // get delta time from TIMER_BEGIN
+ *          TIMER_NOW # get delta time from TIMER_BEGIN
  *          ....
  *      TIMER_END
+ * 
+ * The Timer can be cascaded
+ *
+ *      TIMER_BEGIN # TIMER-1
+ *          ....
+ *          TIMER_BEGIN # TIMER-2
+ *              ....
+ *              TIMER_NOW # delta time from TIMER-2
+ *              ....
+ *          TIMER_END # End of TIMER-2
+ *          ....
+ *          TIMER_NOW # delta time from TIMER-1
+ *          ....
+ *      TIMER_END # End of TIMER-1
  */
 #define TIMER_BEGIN { double __time__ = cv::getTickCount();
 #define TIMER_NOW   ((cv::getTickCount() - __time__) / cv::getTickFrequency())
 #define TIMER_END   }
 
+#define JDA_Assert(expr, msg) assert((expr) && (msg))
+
 namespace jda {
 
 /**
-* Feature used by Cart
-*
-* see more detail on paper in section 4.2
-*/
+ * Feature used by Cart
+ *
+ * see more detail on paper in section 4.2
+ */
 class Feature {
 public:
     static const int ORIGIN = 0;
     static const int HALF = 1;
     static const int QUARTER = 2;
 
+public:
+    /**
+     * Calculate feature value
+     * :input o:        original image
+     * :input h:        half of original image
+     * :input q:        quanter of original image
+     * :input s:        shape of origin image
+     * :return:         feature value
+     *
+     * We have three scaled image and one shape of original image, the shape of half size
+     * and quarter size will be calculated in this function for feature value
+     */
+    int CalcFeatureValue(const cv::Mat& o, const cv::Mat& h, const cv::Mat& q, \
+                         const cv::Mat_<double>& s) const;
+
 public:
     int scale;
     int landmark_id1, landmark_id2;
-    double offset1_x, offset1_y;
+    double offset1_x, offset1_y; // relative offset range in [0, 1]
     double offset2_x, offset2_y;
 
     static inline Feature Default() {
@@ -61,14 +93,19 @@ class Config {
     int K; // number of boost carts in each stage
     int landmark_n; // number of landmarks
     int tree_depth; // depth of cart
-    double tp_rate, fn_rate;
-    int img_width, img_height; // size of all training data
+    double accept_rate, reject_rate;
+    int img_o_width, img_o_height; // size of all training data
+    int img_h_width, img_h_height;
+    int img_q_width, img_q_height;
     int shift_size; // maximum random shift size on mean shape range [0, shift_size]
     double np_ratio; // N(negative) / N(postive)
     std::vector<double> radius; // sample radius of feature points in each stages
     std::vector<int> feats; // feature numbers used by carts in each stages
     std::vector<double> probs; // probability of classification in each stages
 
+    double scale_factor; // hard negative mining parameters
+    int x_step, y_step;
+
     std::string train_txt; // a text file for train dataset
     std::string nega_txt; // a text file for negative dataset
     std::string test_txt; // a text file for test dataset
@@ -81,24 +118,40 @@ class Config {
 };
 
 /**
- * printf with timestamp
+ * Printf with timestamp
  */
 void LOG(const char* fmt, ...);
 
+/**
+ * Terminate the program with a message
+ * **NOTICE** the message shouldn't be too long
+ */
+void dieWithMsg(const char* fmt, ...);
+
 /**
  * Calculate Variance of vector
  */
 double calcVariance(const cv::Mat_<double>& vec);
 double calcVariance(const std::vector<double>& vec);
 
+/**
+ * Calculate Mean Error between gt_shapes and current_shapes
+ */
+double calcMeanError(const std::vector<cv::Mat_<double> >& gt_shapes, \
+                     const std::vector<cv::Mat_<double> >& current_shapes);
+
 /**
  * Check the point (x, y) in Image, modify if needed
+ * :input w:    width of image
+ * :input h:    height of image
+ * :output x:   x of point
+ * :output y:   y of point
  */
-inline void checkBoundaryOfImage(int w, int h, double& x, double& y) {
-    if (x < 0) x = 0.0;
-    if (y < 0) y = 0.0;
-    if (x > w) x = double(w);
-    if (y > h) y = double(h);
+inline void checkBoundaryOfImage(int w, int h, int& x, int& y) {
+    if (x < 0) x = 0;
+    if (y < 0) y = 0;
+    if (x >= w) x = w - 1;
+    if (y >= h) y = w - 1;
 }
 
 } // namespace jda