Updated the transform size and added num workers params

Author: ac-alpha

libreface/AU_Detection/inference.py

@@ -58,7 +58,6 @@ def detect_action_units(image_path:str,
                         'weights_download_id': '17v_vxQ09upLG3Yh0Zlx12rpblP7uoA8x',
                         'data': 'BP4D',
                         'fold': 'all',
-                        'num_workers': 0,
                         'image_size': 256,
                         'crop_size': 224,
                         'num_labels': 12,
@@ -94,6 +93,7 @@ def detect_action_units(image_path:str,
 def detect_action_units_video(aligned_frames_path_list:list, 
                         device:str = "cpu",
                         batch_size:int = 256,
+                        num_workers:int = 2,
                         weights_download_dir = "./weights_libreface")->dict:
 
     opts = ConfigObject({'seed': 0,
@@ -102,7 +102,7 @@ def detect_action_units_video(aligned_frames_path_list:list,
                         'weights_download_id': '17v_vxQ09upLG3Yh0Zlx12rpblP7uoA8x',
                         'data': 'BP4D',
                         'fold': 'all',
-                        'num_workers': 0,
+                        'num_workers': num_workers,
                         'image_size': 256,
                         'crop_size': 224,
                         'num_labels': 12,

libreface/AU_Recognition/inference.py

@@ -63,7 +63,6 @@ def get_au_intensities_and_detect_aus(image_path, device="cpu",
                         'fer_data_root': '',
                         'fer_data': 'AffectNet',
                         'fold': 'all',
-                        'num_workers': 4,
                         'image_size': 256,
                         'crop_size': 224,
                         'au_recognition_num_labels': 12,
@@ -102,7 +101,7 @@ def get_au_intensities_and_detect_aus(image_path, device="cpu",
     return format_output(detected_aus, task="au_detection"), format_output(au_intensities, task = "au_recognition")
 
 def get_au_intensities_and_detect_aus_video(aligned_frames_path, device="cpu",
-                                            batch_size = 256,
+                                            batch_size = 256, num_workers=2,
                                       weights_download_dir="./weights_libreface"):
     opts = ConfigObject({'seed':0,
                         'ckpt_path': f'{weights_download_dir}/AU_Recognition/weights/combined_resnet.pt',
@@ -117,7 +116,7 @@ def get_au_intensities_and_detect_aus_video(aligned_frames_path, device="cpu",
                         'fer_data_root': '',
                         'fer_data': 'AffectNet',
                         'fold': 'all',
-                        'num_workers': 4,
+                        'num_workers': num_workers,
                         'image_size': 256,
                         'crop_size': 224,
                         'au_recognition_num_labels': 12,
@@ -170,7 +169,6 @@ def get_au_intensities(image_path, device="cpu",
                         'weights_download_id': '14qEnWRew2snhdMdOVyqKFJ5rq5VZrfAX',
                         'data': 'DISFA',
                         'fold': 'all',
-                        'num_workers': 0,
                         'image_size': 256,
                         'crop_size': 224,
                         'num_labels': 12,
@@ -204,7 +202,7 @@ def get_au_intensities(image_path, device="cpu",
     return format_output(au_intensities)
 
 def get_au_intensities_video(aligned_frames_path, device="cpu",
-                             batch_size = 256,
+                             batch_size = 256, num_workers = 2,
                         weights_download_dir = "./weights_libreface"):
 
     # # Path to the YAML config file
@@ -218,7 +216,7 @@ def get_au_intensities_video(aligned_frames_path, device="cpu",
                         'weights_download_id': '14qEnWRew2snhdMdOVyqKFJ5rq5VZrfAX',
                         'data': 'DISFA',
                         'fold': 'all',
-                        'num_workers': 0,
+                        'num_workers': num_workers,
                         'image_size': 256,
                         'crop_size': 224,
                         'num_labels': 12,

libreface/Facial_Expression_Recognition/inference.py

@@ -49,7 +49,6 @@ def get_facial_expression(image_path, device = "cpu",
                         'ckpt_path': f'{weights_download_dir}/Facial_Expression_Recognition/weights/resnet.pt',
                         'weights_download_id': '1PeoPj8rga4vU2nuh_PciyX3HqaXp6LP7',
                         'data': 'AffectNet',
-                        'num_workers': 8,
                         'image_size': 224,
                         'num_labels': 8,
                         'dropout': 0.1,
@@ -93,7 +92,7 @@ def get_facial_expression(image_path, device = "cpu",
     return facial_expr_idx_to_class(facial_expression)
 
 def get_facial_expression_video(aligned_frames_path, device = "cpu",
-                                batch_size = 256,
+                                batch_size = 256, num_workers = 2,
                         weights_download_dir = "./weights_libreface"):
     opts = ConfigObject({'seed': 0,
                         'train_csv': 'training_filtered.csv',
@@ -102,7 +101,7 @@ def get_facial_expression_video(aligned_frames_path, device = "cpu",
                         'ckpt_path': f'{weights_download_dir}/Facial_Expression_Recognition/weights/resnet.pt',
                         'weights_download_id': '1PeoPj8rga4vU2nuh_PciyX3HqaXp6LP7',
                         'data': 'AffectNet',
-                        'num_workers': 8,
+                        'num_workers': num_workers,
                         'image_size': 224,
                         'num_labels': 8,
                         'dropout': 0.1,

libreface/__init__.py

@@ -58,6 +58,7 @@ def get_facial_attributes_video(video_path,
                                 temp_dir="./tmp", 
                                 device="cpu",
                                 batch_size = 256,
+                                num_workers = 2,
                                 weights_download_dir:str = "./weights_libreface"):
     print(f"Using device: {device} for inference...")
     frame_extraction_start = time.time()
@@ -83,15 +84,18 @@ def get_facial_attributes_video(video_path,
         detected_aus, au_intensities = get_au_intensities_and_detect_aus_video(aligned_frames_path_list, 
                                                                            device = device, 
                                                                            batch_size=batch_size,
+                                                                           num_workers=num_workers,
                                                                            weights_download_dir=weights_download_dir)
     elif model_choice == "separate_prediction_heads":
         detected_aus = detect_action_units_video(aligned_frames_path_list, 
                                                 device=device,
                                                 batch_size=batch_size,
+                                                num_workers=num_workers,
                                                 weights_download_dir=weights_download_dir)
         au_intensities = get_au_intensities_video(aligned_frames_path_list, 
                                                 device=device,
                                                 batch_size=batch_size,
+                                                num_workers=num_workers,
                                                 weights_download_dir=weights_download_dir)
     else:
         print(f"Undefined model_choice = {model_choice} for get_facial_attributes_video()")
@@ -117,12 +121,14 @@ def save_facial_attributes_video(video_path,
                             temp_dir="./tmp", 
                             device="cpu",
                             batch_size = 256,
+                            num_workers = 2,
                             weights_download_dir:str = "./weights_libreface"):
     frames_df = get_facial_attributes_video(video_path,
                                             model_choice=model_choice,
                                             temp_dir=temp_dir,
                                             device=device, 
                                             batch_size=batch_size,
+                                            num_workers=num_workers,
                                             weights_download_dir=weights_download_dir)
     save_path = uniquify_file(output_save_path)
     frames_df.to_csv(save_path, index=False)
@@ -157,6 +163,7 @@ def get_facial_attributes(file_path,
                           temp_dir="./tmp", 
                           device="cpu",
                           batch_size = 256,
+                          num_workers = 2,
                           weights_download_dir:str = "./weights_libreface"):
     file_type = check_file_type(file_path)
     if file_type == "image":
@@ -175,13 +182,14 @@ def get_facial_attributes(file_path,
         if output_save_path is None:
             return get_facial_attributes_video(file_path, model_choice=model_choice, 
                                                temp_dir=temp_dir, device=device, 
-                                               batch_size=batch_size, weights_download_dir=weights_download_dir)
+                                               batch_size=batch_size, 
+                                               num_workers=num_workers, weights_download_dir=weights_download_dir)
         else:
             try:
                 return save_facial_attributes_video(file_path, output_save_path=output_save_path, 
                                                     model_choice=model_choice, temp_dir=temp_dir, 
                                                     device=device, batch_size=batch_size, 
-                                                    weights_download_dir=weights_download_dir)
+                                                    num_workers=num_workers, weights_download_dir=weights_download_dir)
             except Exception as e:
                 print(e)
                 print("Some error in saving the results.")

libreface/commandline.py

@@ -1,5 +1,4 @@
 import argparse
-import os
 
 from libreface import get_facial_attributes
 
@@ -10,14 +9,16 @@ def main_func():
     parser.add_argument("--output_path", type=str, default="sample_results.csv", help="Path to the csv where results should be saved. Defaults to 'sample_results.csv'")
     parser.add_argument("--device", type=str, default="cpu", help="Device to use while inference. Can be 'cpu', 'cuda:0', 'cuda:1', ... Defaults to 'cpu'")
     parser.add_argument("--temp", type=str, default="./tmp", help="Path where the temporary results for facial attributes can be saved.")
+    parser.add_argument("--num_workers", type=int, default=2, help="Number of workers to be used in the dataloader while doing inference on a video.")
 
     args = parser.parse_args()
 
     get_facial_attributes(args.input_path, 
                           output_save_path=args.output_path, 
                           model_choice="joint_au_detection_intensity_estimator",
                           temp_dir=args.temp, 
-                          device=args.device)
+                          device=args.device,
+                          num_workers=args.num_workers)
 
 if __name__ ==  "__main__":
     main_func()

libreface/detect_mediapipe_image.py

@@ -5,10 +5,7 @@
 import scipy.ndimage
 from PIL import Image, ImageDraw
 from tqdm import tqdm
-from multiprocessing import Process
 import mediapipe as mp
-import pickle
-import time
 import argparse
 
 from libreface.utils import uniquify_file, restructure_landmark_mediapipe
@@ -93,9 +90,10 @@ def image_align(img, face_landmarks, output_size=256,
     quad += pad[:2]
 
   img = img.transform((transform_size, transform_size), Image.Transform.QUAD,
-            (quad + 0.5).flatten(), Image.Resampling.NEAREST)
+            (quad + 0.5).flatten(), Image.Resampling.BILINEAR)
 
-  out_image = img.resize((output_size, output_size), Image.Resampling.NEAREST)
+  out_image = img.resize((output_size, output_size), Image.Resampling.LANCZOS)
+  out_image = img
 
   return out_image
 

test_libreface.py

@@ -1,33 +1,30 @@
 import libreface
 import time
 
-## inference on single image and store results to a variable
-# detected_facial_attributes = libreface.get_facial_attributes_image(image_path = "sample_disfa.png",
-#                                                                    temp_dir = "./temp",
-#                                                                    device = "cuda")
+# inference on single image and store results to a variable
+detected_facial_attributes = libreface.get_facial_attributes_image(image_path = "sample_disfa.png",
+                                                                   temp_dir = "./temp",
+                                                                   device = "cpu")
 
-# ## inference on a single image and save results in a csv file
-# libreface.save_facial_attributes_image(image_path = "sample_disfa.png",
-#                                        output_save_path = "sample_image_results.csv",
-#                                        temp_dir = "./temp",
-#                                        device = "cuda")
+# inference on a single image and save results in a csv file
+libreface.save_facial_attributes_image(image_path = "sample_disfa.png",
+                                       output_save_path = "sample_image_results.csv",
+                                       temp_dir = "./temp",
+                                       device = "cpu")
 
-# ## inference on a video and store the results to a pandas dataframe
-# detected_facial_attributes_df = libreface.get_facial_attributes_video(video_path = "sample_disfa.avi",
-#                                                                       temp_dir = "./temp",
-#                                                                       device = "cuda")
+# inference on a video and store the results to a pandas dataframe
+detected_facial_attributes_df = libreface.get_facial_attributes_video(video_path = "sample_disfa.avi",
+                                                                      temp_dir = "./temp",
+                                                                      device = "cpu")
 
 # ## inference on a video and save the results framewise in a csv file
-start = time.time()
-libreface.save_facial_attributes_video(video_path = "sample_disfa_long.avi",
+libreface.save_facial_attributes_video(video_path = "sample_disfa.avi",
                                        output_save_path = "sample_video_results.csv",
                                        temp_dir = "./temp",
-                                       device = "cuda")
-end = time.time()
-print(f"{end-start:.4f}s - {600 / (end-start):.4f}")
+                                       device = "cpu")
 
-# ## inference on any image or video type and store results accordingly to a variable or save results
-# detected_facial_attributes = libreface.get_facial_attributes(file_path = "sample_disfa.avi",
-#                                                              output_save_path = "sample_results.csv",
-#                                                              temp_dir = "./temp",
-#                                                              device = "cuda")
+## inference on any image or video type and store results accordingly to a variable or save results
+detected_facial_attributes = libreface.get_facial_attributes(file_path = "sample_disfa.avi",
+                                                             output_save_path = "sample_results.csv",
+                                                             temp_dir = "./temp",
+                                                             device = "cpu")