face_mesh_tracker.py

import os
import urllib.request
import sys

import cv2

import mediapipe as mp
from mediapipe.tasks import python
from mediapipe.tasks.python import vision
from mediapipe.framework.formats import landmark_pb2

import time
import numpy as np

# import autopy


class FaceMeshTracker:
    # face bounder indices
    FACE_OVAL = [
        10,
        338,
        297,
        332,
        284,
        251,
        389,
        356,
        454,
        323,
        361,
        288,
        397,
        365,
        379,
        378,
        400,
        377,
        152,
        148,
        176,
        149,
        150,
        136,
        172,
        58,
        132,
        93,
        234,
        127,
        162,
        21,
        54,
        103,
        67,
        109,
    ]

    # lips indices for Landmarks
    LIPS = [
        61,
        146,
        91,
        181,
        84,
        17,
        314,
        405,
        321,
        375,
        291,
        308,
        324,
        318,
        402,
        317,
        14,
        87,
        178,
        88,
        95,
        185,
        40,
        39,
        37,
        0,
        267,
        269,
        270,
        409,
        415,
        310,
        311,
        312,
        13,
        82,
        81,
        42,
        183,
        78,
    ]
    LOWER_LIPS = [
        61,
        146,
        91,
        181,
        84,
        17,
        314,
        405,
        321,
        375,
        291,
        308,
        324,
        318,
        402,
        317,
        14,
        87,
        178,
        88,
        95,
    ]
    UPPER_LIPS = [
        185,
        40,
        39,
        37,
        0,
        267,
        269,
        270,
        409,
        415,
        310,
        311,
        312,
        13,
        82,
        81,
        42,
        183,
        78,
    ]
    # Left eyes indices
    LEFT_EYE = [
        362,
        382,
        381,
        380,
        374,
        373,
        390,
        249,
        263,
        466,
        388,
        387,
        386,
        385,
        384,
        398,
    ]
    LEFT_EYEBROW = [336, 296, 334, 293, 300, 276, 283, 282, 295, 285]
    LEFT_CENTER_EYE = [473]

    # right eyes indices
    RIGHT_EYE = [
        33,
        7,
        163,
        144,
        145,
        153,
        154,
        155,
        133,
        173,
        157,
        158,
        159,
        160,
        161,
        246,
    ]
    RIGHT_EYEBROW = [70, 63, 105, 66, 107, 55, 65, 52, 53, 46]
    RIGHT_CENTER_EYE = [468]

    def __init__(
        self,
        model: str = None,
        num_faces: int = 1,
        min_face_detection_confidence: float = 0.5,
        min_face_presence_confidence: float = 0.5,
        min_tracking_confidence: float = 0.5,
    ):
        """
        Initialize a FaceTracker instance.

        Args:
            model (str): The path to the model for face tracking.
            num_faces (int): Maximum number of faces to detect.
            min_face_detection_confidence (float): Minimum confidence value ([0.0, 1.0]) for successful face detection.
            min_face_presence_confidence (float): Minimum confidence value ([0.0, 1.0]) for presence of a face to be tracked.
            min_tracking_confidence (float): Minimum confidence value ([0.0, 1.0]) for successful face landmark tracking.
        """
        self.model = model

        if self.model == None:
            self.model = self.download_model()

        if self.model == None:
            self.model = self.download_model()

        self.detector = self.initialize_detector(
            num_faces,
            min_face_detection_confidence,
            min_face_presence_confidence,
            min_tracking_confidence,
        )

        self.mp_face_mesh = mp.solutions.face_mesh
        self.mp_drawing = mp.solutions.drawing_utils
        self.mp_drawing_styles = mp.solutions.drawing_styles

        self.DETECTION_RESULT = None

    def save_result(
        self,
        result: vision.FaceLandmarkerResult,
        unused_output_image,
        timestamp_ms: int,
        fps: bool = False,
    ):
        """
        Saves the result of the face detection.

        Args:
            result (vision.FaceLandmarkerResult): Result of the face detection.
            unused_output_image (mp.Image): Unused.
            timestamp_ms (int): Timestamp of the detection.

        Returns:
            None
        """
        self.DETECTION_RESULT = result

    def initialize_detector(
        self,
        num_faces: int,
        min_face_detection_confidence: float,
        min_face_presence_confidence: float,
        min_tracking_confidence: float,
    ):
        """
        Initializes the FaceLandmarker instance.

        Args:
            num_faces (int): Maximum number of faces to detect.
            min_face_detection_confidence (float): Minimum confidence value ([0.0, 1.0]) for face detection to be considered successful.
            min_face_presence_confidence (float): Minimum confidence value ([0.0, 1.0]) for the presence of a face for the face landmarks to be considered tracked successfully.
            min_tracking_confidence (float): Minimum confidence value ([0.0, 1.0]) for the face landmarks to be considered tracked successfully.

        Returns:
            vision.FaceLandmarker: FaceLandmarker instance.
        """
        base_options = python.BaseOptions(model_asset_path=self.model)
        options = vision.FaceLandmarkerOptions(
            base_options=base_options,
            running_mode=vision.RunningMode.LIVE_STREAM,
            num_faces=num_faces,
            min_face_detection_confidence=min_face_detection_confidence,
            min_face_presence_confidence=min_face_presence_confidence,
            min_tracking_confidence=min_tracking_confidence,
            output_face_blendshapes=True,
            result_callback=self.save_result,
        )
        return vision.FaceLandmarker.create_from_options(options)

    def draw_landmarks(
        self,
        image: np.ndarray,
        text_color: tuple = (0, 0, 0),
        font_size: int = 1,
        font_thickness: int = 1,
    ) -> np.ndarray:
        """
        Draws the face landmarks on the image.

        Args:
            image (numpy.ndarray): Image on which to draw the landmarks.
            text_color (tuple, optional): Color of the text. Defaults to (0, 0, 0).
            font_size (int, optional): Size of the font. Defaults to 1.
            font_thickness (int, optional): Thickness of the font. Defaults to 1.

        Returns:
            numpy.ndarray: Image with the landmarks drawn.
        """

        if self.DETECTION_RESULT:
            # Draw landmarks.
            for face_landmarks in self.DETECTION_RESULT.face_landmarks:
                face_landmarks_proto = landmark_pb2.NormalizedLandmarkList()
                face_landmarks_proto.landmark.extend(
                    [
                        landmark_pb2.NormalizedLandmark(
                            x=landmark.x, y=landmark.y, z=landmark.z
                        )
                        for landmark in face_landmarks
                    ]
                )
                self.mp_drawing.draw_landmarks(
                    image=image,
                    landmark_list=face_landmarks_proto,
                    connections=self.mp_face_mesh.FACEMESH_TESSELATION,
                    landmark_drawing_spec=None,
                    connection_drawing_spec=self.mp_drawing_styles.get_default_face_mesh_tesselation_style(),
                )
                self.mp_drawing.draw_landmarks(
                    image=image,
                    landmark_list=face_landmarks_proto,
                    connections=self.mp_face_mesh.FACEMESH_CONTOURS,
                    landmark_drawing_spec=None,
                    connection_drawing_spec=self.mp_drawing_styles.get_default_face_mesh_contours_style(),
                )
                self.mp_drawing.draw_landmarks(
                    image=image,
                    landmark_list=face_landmarks_proto,
                    connections=self.mp_face_mesh.FACEMESH_IRISES,
                    landmark_drawing_spec=None,
                    connection_drawing_spec=self.mp_drawing_styles.get_default_face_mesh_iris_connections_style(),
                )

        return image

    def draw_landmark_circles(
        self,
        image: np.ndarray,
        landmark_indices: list,
        circle_radius: int = 1,
        circle_color: tuple = (0, 255, 0),
        circle_thickness: int = 1,
    ) -> np.ndarray:
        """
        Draws circles on the specified face landmarks on the image.

        Args:
            image (numpy.ndarray): Image on which to draw the landmarks.
            landmark_indices (list of int): Indices of the landmarks to draw.
            circle_radius (int, optional): Radius of the circles. Defaults to 1.
            circle_color (tuple, optional): Color of the circles. Defaults to (0, 255, 0).
            circle_thickness (int, optional): Thickness of the circles. Defaults to 1.

        Returns:
            numpy.ndarray: Image with the landmarks drawn.
        """
        if self.DETECTION_RESULT:
            # Draw landmarks.
            for face_landmarks in self.DETECTION_RESULT.face_landmarks:
                for i, landmark in enumerate(face_landmarks):
                    if i in landmark_indices:
                        # Convert the landmark position to image coordinates.
                        x = int(landmark.x * image.shape[1])
                        y = int(landmark.y * image.shape[0])
                        cv2.circle(
                            image,
                            (x, y),
                            circle_radius,
                            circle_color,
                            circle_thickness,
                        )

        return image

    def detect(self, frame: np.ndarray, draw: bool = False) -> np.ndarray:
        """
        Detects the face landmarks in the frame.

        Args:
            frame (numpy.ndarray): Frame in which to detect the landmarks.
            draw (bool, optional): Whether to draw the landmarks on the frame. Defaults to False.

        Returns:
            numpy.ndarray: Frame with the landmarks drawn.
        """
        rgb_image = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
        mp_image = mp.Image(image_format=mp.ImageFormat.SRGB, data=rgb_image)
        self.detector.detect_async(mp_image, time.time_ns() // 1_000_000)
        return self.draw_landmarks(frame) if draw else frame

    def get_face_landmarks(self, face_idx: int = 0, idxs: list = None) -> list:
        """
        Returns the face landmarks.

        Args:
            face_idx (int, optional): Index of the face for which to return the landmarks. Defaults to 0.
            idxs (list, optional): List of indices of the landmarks to return. Defaults to None.

        Returns:
            list: List of face world landmarks.
        """
        if self.DETECTION_RESULT is not None:
            if idxs is None:
                return self.DETECTION_RESULT.face_landmarks[face_idx]
            else:
                return [
                    self.DETECTION_RESULT.face_landmarks[face_idx][idx] for idx in idxs
                ]
        else:
            return []

    @staticmethod
    def download_model() -> str:
        """
        Download the face_landmarker task model from the mediapipe repository.
            https://storage.googleapis.com/mediapipe-models/face_landmarker/face_landmarker/float16/latest/face_landmarker.task

        Returns:
            str: Path to the downloaded model.
        """
        root = os.path.dirname(os.path.realpath(__file__))
        # Unino to res folder
        root = os.path.join(root, "..", "res")
        filename = os.path.join(root, "face_landmarker.task")
        if os.path.exists(filename):
            print(f"O arquivo {filename} já existe, pulando o download.")
        else:
            base = "https://storage.googleapis.com/mediapipe-models/face_landmarker/face_landmarker/float16/latest/face_landmarker.task"
            urllib.request.urlretrieve(base, filename)

        return filename