diff --git a/cena/FaceDetector.py b/cena/FaceDetector.py deleted file mode 100755 index e42c885..0000000 --- a/cena/FaceDetector.py +++ /dev/null @@ -1,41 +0,0 @@ -# Face detection on the jevois -# We're not using this but keeping it for posterity - -import libjevois as jevois -import cv2 -import os - - -class FaceDetector: - # ################################################################################################### - # Constructor - def __init__(self): - code_path = '/jevois/modules/JeVois/cena' - - cascPath = os.path.join(code_path, 'haarcascade_frontalface_default.xml') - self.faceCascade = cv2.CascadeClassifier(cascPath) - - # ################################################################################################### - # Process function with USB output - def process(self, inframe, outframe): - frame = inframe.getCvGRAY() - # frame = inframe.getCvBGR() - # frame_gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) - - faces = self.faceCascade.detectMultiScale( - frame, - scaleFactor=1.1, - minNeighbors=5, - minSize=(30, 30), - flags=cv2.CASCADE_SCALE_IMAGE - ) - - # if there are faces, send them in a list o lists via serial - if len(faces) > 0: - face_list = [] - for x, y, w, h in faces: - face_list.append([x, y, w, h]) - jevois.sendSerial(str(face_list)) - - # always send the output frame - outframe.sendCvBGR(frame) diff --git a/cena/recognition.py b/cena/recognition.py index 94808bf..4ceefc7 100644 --- a/cena/recognition.py +++ b/cena/recognition.py @@ -10,7 +10,7 @@ from sklearn.svm import SVC from cena.settings import (SHAPE_PREDICTOR_FILE_PATH, CASCADE_FILE_PATH, FEATURE_EXTRACTOR_FILE_PATH, - DEV, LABELS_FILE_PATH, REPS_FILE_PATH, ANNOTATE_FRAME) + DEV, LABELS_FILE_PATH, REPS_FILE_PATH, ANNOTATE_FRAME, TIME_ZONE) class FaceRecognizer(object): @@ -55,8 +55,10 @@ def make_training_set(self, directory='data/img/*', out_dir='data/transformed_im self.output_training_features(file_path, os.path.join(out_dir + file_path.split('/')[-1])) def recognize_faces(self, frame, list_o_faces): - start = datetime.now() - pred_names = [] + start = datetime.now(TIME_ZONE) + pred_names = {} + if frame.shape[-1] != 3: + frame = cv2.cvtColor(frame, cv2.COLOR_GRAY2RGB) for x, y, w, h in list_o_faces: rect = dlib.rectangle(left=x, top=y, right=x+w, bottom=y+h) aligned_face = self.face_aligner.align(96, frame, rect, @@ -68,7 +70,7 @@ def recognize_faces(self, frame, list_o_faces): highest_prob_index = np.argmax(pred_probs) pred_name = self.clf.classes_[highest_prob_index] pred_prob = max(pred_probs) - pred_names.append({pred_name: pred_prob}) + pred_names.update({pred_name: pred_prob}) if ANNOTATE_FRAME: pose_landmarks = self.face_pose_predictor(frame, rect) @@ -78,7 +80,7 @@ def recognize_faces(self, frame, list_o_faces): x, y = point.x, point.y cv2.circle(frame, (x, y), 5, (0, 255, 0), -1) - end = datetime.now() + end = datetime.now(TIME_ZONE) return frame, pred_names, (end - start).microseconds / 1000 # if DEV and ANNOTATE_FRAME: diff --git a/cena/settings.py b/cena/settings.py index 170d2c7..76f419d 100644 --- a/cena/settings.py +++ b/cena/settings.py @@ -1,12 +1,12 @@ import os +import pytz from ast import literal_eval -ENVIRONMENT = 'dev' -# ENVIRONMENT = 'nah dude' +ENVIRONMENT = os.getenv('FACE_ENV', 'lol') DEV = ENVIRONMENT == 'dev' -YOLO_MODE = True -# YOLO_MODE = False +# YOLO_MODE = True +YOLO_MODE = False # ANNOTATE_FRAME = True ANNOTATE_FRAME = False @@ -28,7 +28,6 @@ FEATURE_EXTRACTOR_FILE_NAME = 'nn4.small2.v1.t7' else: FEATURE_EXTRACTOR_FILE_NAME = 'nn4.small2.v1.t7' - # FEATURE_EXTRACTOR_FILE_NAME = 'nn4.small2.v1.ascii.t7' CASCADE_FILE_PATH = os.path.join(MODELS_DIR, CASCADE_FILE_NAME) SHAPE_PREDICTOR_FILE_PATH = os.path.join(MODELS_DIR, SHAPE_PREDICTOR_FILE_NAME) @@ -46,12 +45,16 @@ RYAN_FILE_NAME = 'dun_dun_dun.mp3' RYAN_SONG_PATH = os.path.join(SONGS_DIR, RYAN_FILE_NAME) -if IS_CLIENT: +if not DEV: from cena.utils import get_api_server_ip_address - # SERVER_URL = 'http://localhost:5000/recognize' SERVER_IP = get_api_server_ip_address() - # SERVER_URL = 'http://107.20.57.175:5000/recognize' else: SERVER_IP = 'localhost' SERVER_URL = 'http://{}:5000/recognize'.format(SERVER_IP) + +TIME_ZONE = pytz.timezone('America/Chicago') + +WINDOW_SIZE = 20 +MIN_SEEN = 5 +PROBA_THRESHOLD = 0.4 diff --git a/cena/song_manager.py b/cena/song_manager.py index 828f65d..cacac5a 100644 --- a/cena/song_manager.py +++ b/cena/song_manager.py @@ -1,6 +1,8 @@ +from subprocess import call from glob import glob -from cena.settings import SONGS_DIR +from cena.settings import SONGS_DIR, WINDOW_SIZE, PROBA_THRESHOLD, YOLO_MODE, MIN_SEEN +from cena.utils import play_mp3 def get_name_from_path(file_path): @@ -11,7 +13,63 @@ def get_name_from_path(file_path): class SongManager(object): def __init__(self): - self.song_files = song_files =glob(SONGS_DIR + '*.*') + self.is_blank_slate = True + + self.song_files = song_files = glob(SONGS_DIR + '*.*') self.person_songs = {get_name_from_path(file_name): file_name for file_name in song_files} + self.people = self.person_songs.keys() + + self.person_thresholds = {person: 0. for person in self.people} + self.played_today = self.make_new_slate() + self.window = [] + + def make_new_slate(self): + return {person: 0 for person in self.people} + + def _person_found(self, person): + people_mask = [p == person for p in self.window] + total_found = sum(people_mask) + + more_than_half = total_found >= int(WINDOW_SIZE / 2) + half_of_seen = total_found >= int(len(self.window) / 2) + more_than_min = total_found > MIN_SEEN + return more_than_half and more_than_min and half_of_seen + + def update_window(self, person, proba): + if proba > PROBA_THRESHOLD: + self.window.append(person) + if len(self.window) > WINDOW_SIZE: + self.window.pop(0) + + if self._person_found(person): + # print(self.window) + try: + self.go_song_go(person) + except KeyError as error: + print('oh whoops no song for {}'.format(person)) + + def go_song_go(self, person): + if self.played_today[person] < 1: + print('playing that funky music for {}'.format(person)) + play_mp3(self.person_songs[person]) + self.window = [] + + if not YOLO_MODE: + self.played_today[person] = 1 + else: + # print('you\'ve already had your fill today {}'.format(person)) + pass + + def blank_the_slate(self): + if self.is_blank_slate: + return + self.played_today = self.make_new_slate() + self.window = [] + print('oh wow such reset') + # may not be the right place, but don't want to forget + def update_dropbox(self): + # fixme: make this the right command + command = "/home/pi/Dropbox-Uploader/dropbox_uploader.sh download" + call([command], shell=True) \ No newline at end of file diff --git a/cena/utils.py b/cena/utils.py index 41dc375..7ec5d2d 100644 --- a/cena/utils.py +++ b/cena/utils.py @@ -18,6 +18,7 @@ def decode_image(encoded_str, shape): def play_mp3(path): process = subprocess.Popen(['mpg123', '-q', path]) + process.wait() def get_api_server_id(): @@ -70,120 +71,3 @@ def start_instance(instance): response = instance.start() instance.wait_until_running() print('Instance started at {}'.format(instance.public_ip_address)) - -# -# class EC2Manager(object): -# def __init__(self, config_path=None, start_if_stopped=True, stop_if_started=False): -# self.config = config = Config(config_path) -# self.storm_name, self.storm_enabled = config.storm_name, config.storm_installed -# self.project_name, self.user_name = config.project_name, config.user_name -# -# self.ec2_client = boto3.client('ec2') -# self.ec2_manager = boto3.resource('ec2') -# -# self.instance_id = self.get_instance_id_from_project_name() -# self.instance = instance = self.ec2_manager.Instance(self.instance_id) -# if start_if_stopped: -# self.start_if_not_started() -# elif stop_if_started: -# self.stop_if_not_stopped() -# return -# -# self.instance_ip = instance.public_ip_address -# -# self.public_key_name, self.public_key_path, self.public_key = self.get_public_key() -# -# if self.storm_name and self.storm_enabled: -# self.update_storm() -# -# def start_if_not_started(self): -# if self.instance.state['Name'] in ['stopped', 'stopping']: -# self.start_instance() -# else: -# self.instance_ip = self.instance.public_ip_address -# print(f'instance already started at {self.instance_ip}') -# -# def stop_if_not_stopped(self): -# state = self.instance.state['Name'] -# if state in ['pending', 'running']: -# self.stop_instance() -# else: -# print('instance already stopped or is stopping!') -# -# def terminate_instance(self): -# print(f'alrighty then, terminating instance {self.instance_id}...') -# self.instance.terminate() -# self.instance.wait_until_terminated() -# print('instance terminated') -# -# def start_instance(self): -# print(f'Starting instance {self.instance_id}...') -# response = self.instance.start() -# self.instance.wait_until_running() -# print(f'Instance started at {self.instance.public_ip_address}') -# -# def stop_instance(self): -# print(f'Stopping instance {self.instance_id}...') -# response = self.instance.stop() -# self.instance.wait_until_stopped() -# print('Instance stopped') -# -# def update_storm(self): -# print('Fixin\' up a storm...') -# storm = Storm() -# if storm.is_host_in(self.storm_name, regexp_match=True): -# print('Updating storm profile with latest instance ip') -# storm_update( -# name=self.storm_name, -# connection_uri=f'ubuntu@{self.instance_ip}', -# id_file=self.public_key_path, -# o=['LocalForward=8889 127.0.0.1:8888'] -# ) -# else: -# print('Creating storm profile') -# storm_add( -# name=self.storm_name, -# connection_uri=f'ubuntu@{self.instance_ip}', -# id_file=self.public_key_path, -# o=['LocalForward=8889 127.0.0.1:8888'] -# ) -# print('Storm updated') -# -# def get_instance_id_from_project_name(self): -# filters = [ -# { -# 'Name': 'tag:created_by', -# 'Values': [self.user_name] -# }, -# { -# 'Name': 'tag:project_name', -# 'Values': [self.project_name] -# }, -# { -# 'Name': 'instance-state-name', -# 'Values': ['pending', 'running', 'stopping', 'stopped'] -# } -# ] -# -# response = self.ec2_client.describe_instances(Filters=filters) -# instances = response['Reservations'] -# if instances: -# instance_id = instances[0]['Instances'][0]['InstanceId'] -# return instance_id -# else: -# print('No instances found!') -# return None -# -# def get_public_key(self): -# public_key_name = self.instance.key_name -# public_key_path = os.path.join(os.path.expanduser('~'), '.ssh', f'{public_key_name}.pem') -# public_key = paramiko.RSAKey.from_private_key_file(public_key_path) -# print(f'Using public key at {public_key_path}') -# return public_key_name, public_key_path, public_key -# -# def resize_instance(self, new_size): -# self.stop_if_not_stopped() -# print(f'changing instance to type {new_size}...') -# self.ec2_client.modify_instance_attribute(InstanceId=self.instance_id, Attribute='instanceType', Value=new_size) -# print(f'instance type changed') -# self.start_if_not_started() \ No newline at end of file diff --git a/check_songs.py b/check_songs.py deleted file mode 100644 index 7eee91e..0000000 --- a/check_songs.py +++ /dev/null @@ -1,3 +0,0 @@ -from cena.song_manager import SongManager - -song_manager = SongManager() diff --git a/face_detector.py b/face_detector.py index f3e11cc..ae291f7 100755 --- a/face_detector.py +++ b/face_detector.py @@ -3,7 +3,7 @@ from datetime import datetime from cena.recognition import FaceRecognizer -from cena.settings import DEV, ANNOTATE_FRAME, CASCADE_FILE_PATH, SERVER_URL +from cena.settings import DEV, ANNOTATE_FRAME, CASCADE_FILE_PATH, SERVER_URL, TIME_ZONE from cena.song_manager import SongManager from cena.utils import encode_image, decode_image, play_mp3 @@ -14,23 +14,23 @@ def listen_for_quit(): return True -def get_server_response(frame, list_o_faces): +def get_server_response(frame, list_o_faces, return_frame=False): # response = post(SERVER_URL, json={'list_o_faces': list_o_faces, 'frame': frame.tolist()}) # , files=files) shape = frame.shape request_json = { 'list_o_faces': list_o_faces, 'frame': encode_image(frame), - 'shape': shape + 'shape': shape, + 'return_frame': return_frame } response = post(SERVER_URL, json=request_json) + people_list = response.json()['people_list'] time = response.json()['time'] - if ANNOTATE_FRAME and DEV: + if return_frame: frame = decode_image(response.json()['frame'], shape) - return frame, people_list, time - else: - return people_list, time + return frame, people_list, time # return response.json()['frame'], response.json()['people_list'], response.json()['time'] @@ -38,34 +38,41 @@ def process_frame(video_capture, face_recognizer=None): if not video_capture.grab(): return try: - now = datetime.now() - ret, frame = video_capture.read(cv2.COLOR_BGR2GRAY) + now = datetime.now(TIME_ZONE) + if now.hour > 21: + song_manager.blank_the_slate() + + ret, frame = video_capture.read() + frame = cv2.cvtColor(frame, cv2.COLOR_RGB2GRAY) + # ret, frame = video_capture.read(cv2.COLOR_BGR2GRAY) + # ret, frame = video_capture.read(cv2.IMREAD_GRAYSCALE) frame_gray = frame # frame_gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) + # frame = frame_gray faces = face_cascade.detectMultiScale( frame_gray, - scaleFactor=1.1, + # scaleFactor=1.1, minNeighbors=5, minSize=(80, 80), - flags=cv2.CASCADE_SCALE_IMAGE + # minSize=(60, 60), + # flags=cv2.CASCADE_SCALE_IMAGE ) if len(faces) > 0: - # RYAN_PLAYED = play_mp3(RYAN_SONG_PATH, RYAN_PLAYED) list_o_faces = [] for x, y, w, h in faces: list_o_faces.append([int(x), int(y), int(w), int(h)]) - if DEV and ANNOTATE_FRAME: - # frame, people_list, time = face_recognizer.recognize_faces(frame, list_o_faces) - frame, people_list, time = get_server_response(frame, list_o_faces) - elif DEV: - people_list, time = get_server_response(frame, list_o_faces) - # people_list, time = face_recognizer.recognize_faces(frame, list_o_faces) + if DEV: + frame, people_list, time = face_recognizer.recognize_faces(frame, list_o_faces) + # frame, people_list, time = get_server_response(frame, list_o_faces, ANNOTATE_FRAME) else: - people_list, time = get_server_response(frame, list_o_faces) - # play_mp3(RYAN_SONG_PATH) - print(people_list, datetime.now() - now) + frame, people_list, time = get_server_response(frame, list_o_faces, ANNOTATE_FRAME) + for person, proba in people_list.items(): + song_played = song_manager.update_window(person, proba) + if song_played: + print(people_list, datetime.now(TIME_ZONE) - now) else: - print(datetime.now() - now) + pass + # print(datetime.now(TIME_ZONE) - now) if DEV: # Display the resulting frame @@ -93,12 +100,15 @@ def process_frame(video_capture, face_recognizer=None): print([i for i in trained_people if i not in person_songs]) video_capture = cv2.VideoCapture(1) + video_capture.set(cv2.CAP_PROP_FPS, 5) video_capture.set(cv2.CAP_PROP_FRAME_WIDTH, 320) video_capture.set(cv2.CAP_PROP_FRAME_HEIGHT, 240) + # video_capture.set(cv2.CAP_PROP_FRAME_WIDTH, 400) + # video_capture.set(cv2.CAP_PROP_FRAME_HEIGHT, 300) else: video_capture = cv2.VideoCapture(0) - video_capture.set(cv2.cv.CV_CAP_PROP_FPS, 25) + video_capture.set(cv2.cv.CV_CAP_PROP_FPS, 5) # video_capture.set(cv2.cv.CV_CAP_PROP_FRAME_WIDTH, 640) # video_capture.set(cv2.cv.CV_CAP_PROP_FRAME_HEIGHT, 480) video_capture.set(cv2.cv.CV_CAP_PROP_FRAME_WIDTH, 320) diff --git a/feature_server.py b/feature_server.py index 30cf3a8..8de278c 100644 --- a/feature_server.py +++ b/feature_server.py @@ -30,6 +30,7 @@ def recognize(): # print(frame[0][0].dtype) encoded_frame = request.json['frame'] shape = request.json['shape'] + return_frame = request.json.get('return_frame', False) frame = decode_image(encoded_frame, shape) list_o_faces = request.json['list_o_faces'] @@ -44,7 +45,7 @@ def recognize(): 'time': time } - if DEV and ANNOTATE_FRAME: + if return_frame: response.update({'frame': encode_image(frame)}) return jsonify(response)