Modified key point tracking

cpp/inspireface/common/face_data/face_serialize_tools.h

@@ -10,6 +10,7 @@
 #include "../face_info/face_object_internal.h"
 #include "herror.h"
 #include "data_type.h"
+#include "track_module/landmark/all.h"
 
 // Define the namespace "inspire" for encapsulation
 namespace inspire {
@@ -100,7 +101,7 @@ inline HyperFaceData INSPIRE_API FaceObjectInternalToHyperFaceData(const FaceObj
     if (!obj.landmark_smooth_aux_.empty()) {
         data.densityLandmarkEnable = 1;
         const auto& lmk = obj.landmark_smooth_aux_.back();
-        for (size_t i = 0; i < lmk.size(); i++) {
+        for (size_t i = 0; i < FaceLandmarkAdapt::NUM_OF_LANDMARK; i++) {
             data.densityLandmark[i].x = lmk[i].GetX();
             data.densityLandmark[i].y = lmk[i].GetY();
         }

cpp/inspireface/common/face_info/face_object_internal.h

@@ -26,15 +26,17 @@ class INSPIRE_API FaceObjectInternal {
         pose_euler_angle_.resize(3);
         keyPointFive.resize(5);
         face_action_ = std::make_shared<FaceActionPredictor>(10);
+        num_of_dense_landmark_ = num_landmark;
     }
 
-    void SetLandmark(const std::vector<inspirecv::Point2f> &lmk, bool update_rect = true, bool update_matrix = true, float h = 0.06f, int n = 5) {
-        if (lmk.size() != landmark_.size()) {
-            INSPIRE_LOGW("The SetLandmark function displays an exception indicating that the lmk number does not match");
-            return;
-        }
+    void SetLandmark(const std::vector<inspirecv::Point2f> &lmk, bool update_rect = true, bool update_matrix = true, float h = 0.06f, int n = 5,
+                     int num_of_lmk = 106 * 2) {
+        // if (lmk.size() != landmark_.size()) {
+        //     INSPIRE_LOGW("The SetLandmark function displays an exception indicating that the lmk number does not match");
+        //     return;
+        // }
         std::copy(lmk.begin(), lmk.end(), landmark_.begin());
-        DynamicSmoothParamUpdate(landmark_, landmark_smooth_aux_, 106 * 2, h, n);
+        DynamicSmoothParamUpdate(landmark_, landmark_smooth_aux_, num_of_lmk, h, n);
         // std::cout << "smooth ratio: " << h << " num smooth cache frame: " << n << std::endl;
 
         // cv::Vec3d euler_angle;
@@ -202,6 +204,8 @@ class INSPIRE_API FaceObjectInternal {
     inspirecv::Vec3f euler_angle_;
     std::vector<float> pose_euler_angle_;
 
+    int num_of_dense_landmark_;
+
     float align_mse_{};
 
     const inspirecv::Vec3f &getEulerAngle() const {

cpp/inspireface/track_module/face_track_module.cpp

@@ -73,7 +73,7 @@ bool FaceTrackModule::TrackFace(inspirecv::InspireImageProcess &image, FaceObjec
     // If it is a detection state, calculate the affine transformation matrix
     if (face.TrackingState() == ISF_DETECT) {
         COST_TIME_SIMPLE(GetRectSquare);
-        inspirecv::Rect2i rect_square = face.GetRectSquare(0.0);
+        inspirecv::Rect2i rect_square = face.GetRectSquare(0);
 
         std::vector<inspirecv::Point2f> rect_pts = rect_square.As<float>().ToFourVertices();
         inspirecv::TransformMatrix rotation_mode_affine = image.GetAffineMatrix();
@@ -108,6 +108,7 @@ bool FaceTrackModule::TrackFace(inspirecv::InspireImageProcess &image, FaceObjec
         auto affine_extensive = face.getTransMatrixExtensive();
         auto pre_crop = image.ExecuteImageAffineProcessing(affine_extensive, m_crop_extensive_size_, m_crop_extensive_size_);
         auto res = (*m_face_quality_)(pre_crop);
+
         auto affine_extensive_inv = affine_extensive.GetInverse();
         std::vector<inspirecv::Point2f> lmk_extensive = ApplyTransformToPoints(res.lmk, affine_extensive_inv);
         res.lmk = lmk_extensive;
@@ -186,22 +187,21 @@ bool FaceTrackModule::TrackFace(inspirecv::InspireImageProcess &image, FaceObjec
                 // INSPIRE_LOGD("Extensive Affine Cost %f", extensive_cost_time.GetCostTimeUpdate());
             }
         }
-        // Replace the landmark with the high-quality landmark
-        landmark_back[FaceLandmarkAdapt::LEFT_EYE_CENTER] = face.high_result.lmk[0];
-        landmark_back[FaceLandmarkAdapt::RIGHT_EYE_CENTER] = face.high_result.lmk[1];
-        landmark_back[FaceLandmarkAdapt::NOSE_CORNER] = face.high_result.lmk[2];
-        landmark_back[FaceLandmarkAdapt::MOUTH_LEFT_CORNER] = face.high_result.lmk[3];
-        landmark_back[FaceLandmarkAdapt::MOUTH_RIGHT_CORNER] = face.high_result.lmk[4];
+        // Add five key points to landmark_back
+        for (int i = 0; i < 5; i++) {
+            landmark_back.push_back(face.high_result.lmk[i]);
+        }
         // Update face key points
-        face.SetLandmark(landmark_back, true, true, m_track_mode_smooth_ratio_, m_track_mode_num_smooth_cache_frame_);
+        face.SetLandmark(landmark_back, true, true, m_track_mode_smooth_ratio_, m_track_mode_num_smooth_cache_frame_,
+                         (FaceLandmarkAdapt::NUM_OF_LANDMARK + 10) * 2);
         // Get the smoothed landmark
         auto &landmark_smooth = face.landmark_smooth_aux_.back();
         // Update the face key points
-        face.high_result.lmk[0] = landmark_smooth[FaceLandmarkAdapt::LEFT_EYE_CENTER];
-        face.high_result.lmk[1] = landmark_smooth[FaceLandmarkAdapt::RIGHT_EYE_CENTER];
-        face.high_result.lmk[2] = landmark_smooth[FaceLandmarkAdapt::NOSE_CORNER];
-        face.high_result.lmk[3] = landmark_smooth[FaceLandmarkAdapt::MOUTH_LEFT_CORNER];
-        face.high_result.lmk[4] = landmark_smooth[FaceLandmarkAdapt::MOUTH_RIGHT_CORNER];
+        face.high_result.lmk[0] = landmark_smooth[FaceLandmarkAdapt::NUM_OF_LANDMARK + 0];
+        face.high_result.lmk[1] = landmark_smooth[FaceLandmarkAdapt::NUM_OF_LANDMARK + 1];
+        face.high_result.lmk[2] = landmark_smooth[FaceLandmarkAdapt::NUM_OF_LANDMARK + 2];
+        face.high_result.lmk[3] = landmark_smooth[FaceLandmarkAdapt::NUM_OF_LANDMARK + 3];
+        face.high_result.lmk[4] = landmark_smooth[FaceLandmarkAdapt::NUM_OF_LANDMARK + 4];
     }
 
     // If tracking status, update the confidence level
@@ -315,7 +315,7 @@ void FaceTrackModule::DetectFace(const inspirecv::Image &input, float scale) {
         std::vector<STrack> output_stracks = m_TbD_tracker_->update(objects);
         for (const auto &st_track : output_stracks) {
             inspirecv::Rect<int> rect = inspirecv::Rect<int>(st_track.tlwh[0], st_track.tlwh[1], st_track.tlwh[2], st_track.tlwh[3]);
-            FaceObjectInternal faceinfo(st_track.track_id, rect, FaceLandmarkAdapt::NUM_OF_LANDMARK);
+            FaceObjectInternal faceinfo(st_track.track_id, rect, FaceLandmarkAdapt::NUM_OF_LANDMARK + 10);
             faceinfo.detect_bbox_ = rect.As<int>();
             candidate_faces_.push_back(faceinfo);
         }
@@ -336,7 +336,7 @@ void FaceTrackModule::DetectFace(const inspirecv::Image &input, float scale) {
                 tracking_idx_ = tracking_idx_ + 1;
             }
 
-            FaceObjectInternal faceinfo(tracking_idx_, bbox[i], FaceLandmarkAdapt::NUM_OF_LANDMARK);
+            FaceObjectInternal faceinfo(tracking_idx_, bbox[i], FaceLandmarkAdapt::NUM_OF_LANDMARK + 10);
             faceinfo.detect_bbox_ = bbox[i];
             faceinfo.SetConfidence(boxes[i].score);
 

cpp/inspireface/track_module/face_track_module.h

@@ -200,7 +200,8 @@ class INSPIRE_API FaceTrackModule {
     int m_dynamic_detection_input_level_ = -1;    ///< Detector size class for dynamic input.
 
     float m_crop_extensive_ratio_ = 1.8f;  ///< Crop extensive ratio
-    int m_crop_extensive_size_ = 96;       ///< Crop extensive size
+    // float m_crop_extensive_ratio_ = 1.5f;  ///< Crop extensive ratio
+    int m_crop_extensive_size_ = 96;  ///< Crop extensive size
 
     DetectModuleMode m_mode_;  ///< Detect mode
 

cpp/sample/source/tracker_sample.cpp

@@ -9,7 +9,7 @@ int main() {
     std::string expansion_path = "";
     INSPIRE_LAUNCH->Load("test_res/pack/Pikachu");
     auto archive = INSPIRE_LAUNCH->getMArchive();
-    auto mode = inspire::DetectModuleMode::DETECT_MODE_LIGHT_TRACK;
+    auto mode = inspire::DetectModuleMode::DETECT_MODE_ALWAYS_DETECT;
     FaceTrackModule tracker(mode, 10, 20, 320, -1);
     tracker.Configuration(archive, expansion_path);
 

doc/Error-Feedback-Codes.md

@@ -1,5 +1,6 @@
- # Error Feedback Codes
- During the use of InspireFace, some error feedback codes may be generated. Here is a table of error feedback codes.
+# Error Feedback Codes
+
+During the use of InspireFace, some error feedback codes may be generated. Here is a table of error feedback codes.
 
  | Index | Name | Code | Comment | 
  | --- | --- | --- | --- | 

python/README.md

@@ -13,13 +13,23 @@ pip install inspireface
 
 ## Setup Library
 
+#### Copy the compiled dynamic library
+
 You need to compile the dynamic linking library in the main project and then place it in **inspireface/modules/core/SYSTEM/CORE_ARCH/**.
 
 ```Bash
 # copy or link
 cp YOUR_BUILD_DIR/libInspireFace.so inspireface/modules/core/SYSTEM/CORE_ARCH/
 ```
 
+#### Install
+
+Run the command to install: 
+
+```
+python setup.py install
+```
+
 ## Require
 
 You need to install some dependencies beforehand.
@@ -38,14 +48,11 @@ You can easily call the api to implement a number of functions:
 import cv2
 import inspireface as isf
 
-# Step 1: Initialize the SDK and load the algorithm resource files.
-resource_path = "pack/Pikachu"
-ret = isf.launch(resource_path)
-assert ret, "Launch failure. Please ensure the resource path is correct."
-
 # Optional features, loaded during session creation based on the modules specified.
 opt = isf.HF_ENABLE_NONE
-session = isf.InspireFaceSession(opt, isf.HF_DETECT_MODE_IMAGE)
+session = isf.InspireFaceSession(opt, isf.HF_DETECT_MODE_ALWAYS_DETECT)
+# Set detection confidence threshold
+session.set_detection_confidence_threshold(0.5)
 
 # Load the image using OpenCV.
 image = cv2.imread(image_path)
@@ -60,14 +67,30 @@ draw = image.copy()
 for idx, face in enumerate(faces):
     print(f"{'==' * 20}")
     print(f"idx: {idx}")
-    # Print detection confidence.
     print(f"detection confidence: {face.detection_confidence}")
     # Print Euler angles of the face.
     print(f"roll: {face.roll}, yaw: {face.yaw}, pitch: {face.pitch}")
-    # Draw bounding box around the detected face.
+
+    # Get face bounding box
     x1, y1, x2, y2 = face.location
-    cv2.rectangle(draw, (x1, y1), (x2, y2), (0, 0, 255), 2)
 
+    # Calculate center, size, and angle
+    center = ((x1 + x2) / 2, (y1 + y2) / 2)
+    size = (x2 - x1, y2 - y1)
+    angle = face.roll
+
+    # Apply rotation to the bounding box corners
+    rect = ((center[0], center[1]), (size[0], size[1]), angle)
+    box = cv2.boxPoints(rect)
+    box = box.astype(int)
+
+    # Draw the rotated bounding box
+    cv2.drawContours(draw, [box], 0, (100, 180, 29), 2)
+
+    # Draw landmarks
+    lmk = session.get_face_dense_landmark(face)
+    for x, y in lmk.astype(int):
+        cv2.circle(draw, (x, y), 0, (220, 100, 0), 2)
 ```
 
 

python/inspireface/modules/inspireface.py

@@ -335,6 +335,20 @@ def face_detection(self, image) -> List[FaceInformation]:
             return infos
         else:
             return []
+
+    def get_face_five_key_points(self, single_face: FaceInformation):
+        num_landmarks = 5
+        landmarks_array = (HPoint2f * num_landmarks)()
+        ret = HFGetFaceFiveKeyPointsFromFaceToken(single_face._token, landmarks_array, num_landmarks)
+        if ret != 0:
+            logger.error(f"An error occurred obtaining a dense landmark for a single face: {ret}")
+
+        landmark = []
+        for point in landmarks_array:
+            landmark.append(point.x)
+            landmark.append(point.y)
+
+        return np.asarray(landmark).reshape(-1, 2)
 
     def get_face_dense_landmark(self, single_face: FaceInformation):
         num_landmarks = HInt32()

python/sample_face_track_from_video.py

@@ -16,6 +16,10 @@ def generate_color(id):
     Returns:
         tuple: A tuple representing the color in BGR format.
     """
+    # Handle invalid ID (-1)
+    if id < 0:
+        return (128, 128, 128)  # Return gray color for invalid ID
+
     max_id = 50  # Number of unique colors
     id = id % max_id
 
@@ -119,6 +123,10 @@ def case_face_tracker_from_video(source, show, out):
             for x, y in lmk.astype(int):
                 cv2.circle(frame, (x, y), 0, color, 4)
 
+            five_key_points = session.get_face_five_key_points(face)
+            for x, y in five_key_points.astype(int):
+                cv2.circle(frame, (x, y), 0, (255-color[0], 255-color[1], 255-color[2]), 6)
+
             # Draw track ID at the top of the bounding box
             text = f"ID: {face.track_id}"
             text_size, _ = cv2.getTextSize(text, cv2.FONT_HERSHEY_SIMPLEX, 0.6, 2)

tools/error_table.md

@@ -1,3 +1,5 @@
+# Error Feedback Codes
+During the use of InspireFace, some error feedback codes may be generated. Here is a table of error feedback codes.
  | Index | Name | Code | Comment | 
  | --- | --- | --- | --- | 
  | 1 | HSUCCEED | 0 | Success | 

tools/output_error_table.py

@@ -76,8 +76,14 @@ def process_header(header_path, output_path):
     # Parse and calculate the error codes from the header content
     parsed_error_codes = parse_and_calculate_error_codes(header_content)
 
+    md_table = """# Error Feedback Codes
+
+During the use of InspireFace, some error feedback codes may be generated. Here is a table of error feedback codes.
+
+"""
+
     # Prepare the Markdown table header
-    md_table = " | Index | Name | Code | Comment | \n"
+    md_table += " | Index | Name | Code | Comment | \n"
     md_table += " | --- | --- | --- | --- | \n"
 
     # Fill the Markdown table with parsed error codes