Merge branch 'master' of http://khuhub.khu.ac.kr/2021-1-capstone-design2/2017103972

- Subproject commit 0066cd8581e59918a924357d3371b1e1298cd0e5

+import dlib, cv2
+import numpy as np
+
+detector = dlib.get_frontal_face_detector()
+sp = dlib.shape_predictor('models/shape_predictor_68_face_landmarks.dat')
+facerec = dlib.face_recognition_model_v1('models/dlib_face_recognition_resnet_model_v1.dat')
+
+descs = np.load('img/descs.npy', allow_pickle=True)[()]
+
+def encode_face(img):
+  dets = detector(img, 1)
+
+  if len(dets) == 0:
+    return np.empty(0)
+
+  for k, d in enumerate(dets):
+    shape = sp(img, d)
+    face_descriptor = facerec.compute_face_descriptor(img, shape)
+
+    return np.array(face_descriptor)
+
+video_path = './data/record0.mp4'
+cap = cv2.VideoCapture(video_path)
+
+if not cap.isOpened():
+  exit()
+
+_, img_bgr = cap.read() # (800, 1920, 3)
+padding_size = 0
+resized_width = 1920
+video_size = (resized_width, int(img_bgr.shape[0] * resized_width // img_bgr.shape[1]))
+output_size = (resized_width, int(img_bgr.shape[0] * resized_width // img_bgr.shape[1] + padding_size * 2))
+
+fourcc = cv2.VideoWriter_fourcc('m', 'p', '4', 'v')
+writer = cv2.VideoWriter('%s_output.mp4' % (video_path.split('.')[0]), fourcc, cap.get(cv2.CAP_PROP_FPS), output_size)
+
+while True:
+  ret, img_bgr = cap.read()
+  if not ret:
+    break
+
+  img_bgr = cv2.resize(img_bgr, video_size)
+  img_rgb = cv2.cvtColor(img_bgr, cv2.COLOR_BGR2RGB)
+
+  # img_bgr = cv2.copyMakeBorder(img_bgr, top=padding_size, bottom=padding_size, left=0, right=0, borderType=cv2.BORDER_CONSTANT, value=(0,0,0))
+  
+  dets = detector(img_bgr, 1)
+
+  for k, d in enumerate(dets):
+    shape = sp(img_rgb, d)
+    face_descriptor = facerec.compute_face_descriptor(img_rgb, shape)
+
+    last_found = {'name': 'unknown', 'dist': 0.6, 'color': (0,0,255)}
+
+    for name, saved_desc in descs.items():
+      dist = np.linalg.norm([face_descriptor] - saved_desc, axis=1)
+
+      if dist < last_found['dist']:
+        last_found = {'name': name, 'dist': dist, 'color': (255,255,255)}
+
+    cv2.rectangle(img_bgr, pt1=(d.left(), d.top()), pt2=(d.right(), d.bottom()), color=last_found['color'], thickness=2)
+    cv2.putText(img_bgr, last_found['name'], org=(d.left(), d.top()), fontFace=cv2.FONT_HERSHEY_SIMPLEX, fontScale=1, color=last_found['color'], thickness=2)
+
+  writer.write(img_bgr)
+
+  cv2.imshow('img', img_bgr)
+  
+  if cv2.waitKey(1) == ord('q'):
+    break
+
+cap.release()
+writer.release()

+import dlib
+import cv2
+import numpy as np
+import matplotlib.pyplot as plt
+import tensorflow as tf
+import math
+import os
+import pathlib
+import time
+import pandas as pd
+import tensorflow as tf
+from tensorflow.keras.preprocessing.image import ImageDataGenerator,load_img
+from tensorflow.keras.models import load_model
+from tensorflow.keras import regularizers
+from tensorflow import keras
+from imutils import face_utils
+import time
+
+ 
+start = time.time()
+detector = dlib.get_frontal_face_detector()
+predictor = dlib.shape_predictor("./models/shape_predictor_68_face_landmarks.dat")
+facerec = dlib.face_recognition_model_v1('models/dlib_face_recognition_resnet_model_v1.dat')
+model = load_model('../checkpoint/er-best-mobilenet1-bt32-model-classweight-adam.h5')
+# model = load_model('../checkpoint/er-best-mobilenet2-bt32-model-adam.h5')
+# model = load_model('../checkpoint/er-best-efficientNet1-bt32-model-SGD.h5')
+    
+    
+descs = np.load('img/descs2.npy', allow_pickle=True)[()]
+ 
+video_path = './data/zoom_1.mp4'
+cap=cv2.VideoCapture(video_path)
+ 
+
+
+labels_dict_ = {0 : 'angry', 1 : 'fear' ,  2: 'happy', 3: 'neutral', 4:  'sad', 5: 'surprise'}
+# labels_dict_ = {'angry' : 0,'fear' : 1 ,'happy' : 2, 'neutral' : 3,  'sad' : 4, 'surprise' : 5}
+time_dict = {'angry': [], 'fear': [], 'happy': [], 'neutral': [], 'sad': [], 'surprise': []}
+def get_key(val):
+    for key, value in labels_dict_.items():
+        if(value == val):
+            return key
+
+
+def convertMillis(millis):
+    seconds=(millis/1000)%60 
+    minutes=(millis/(1000*60))%60
+    hours=(millis/(1000*60*60))%24
+    return seconds, int(minutes), int(hours)
+
+#cap = cv2.VideoCapture(0) # 0번 카메라
+ 
+# 동영상 크기(frame정보)를 읽어옴
+frameWidth = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
+frameHeight = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
+frame_size = (frameWidth, frameHeight)
+fps = cap.get((cv2.CAP_PROP_FPS))
+
+
+_, img_bgr = cap.read() # (800, 1920, 3)
+padding_size = 0
+resized_width = 1920
+video_size = (resized_width, int(img_bgr.shape[0] * resized_width // img_bgr.shape[1]))
+timestamps = [cap.get(cv2.CAP_PROP_POS_MSEC)]
+prev_time = 0
+
+fourcc = cv2.VideoWriter_fourcc('m', 'p', '4', 'v')
+# out1 = cv2.VideoWriter('./data/record0.mp4',fourcc, fps, frame_size)
+
+while True:
+    retval, frameBGR = cap.read()	# 영상을 한 frame씩 읽어오기
+    current_time = time.time() - prev_time
+
+    if(type(frameBGR) == type(None)):
+        pass
+    else:
+        frameBGR = cv2.resize(frameBGR, video_size)
+        frame = cv2.cvtColor(frameBGR, cv2.COLOR_BGR2RGB)
+        
+        if (retval is True) and (current_time > 1.5) :
+            prev_time = time.time()
+            faces = detector(frame, 1)
+            
+            for (i, face) in enumerate(faces):
+                shape = predictor(frame, face)
+                face_descriptor = facerec.compute_face_descriptor(frame, shape)
+                
+                img = cv2.resize(frame[face.top():face.bottom(), face.left():face.right()], dsize=(224, 224), interpolation = cv2.INTER_CUBIC)
+                imgarr = np.array(img).reshape(1, 224, 224, 3) /255
+                emotion = labels_dict_[model.predict(imgarr).argmax(axis=-1)[0]]
+                # emotion = get_key(model.predict_classes(imgarr))
+
+                
+                last_found = {'name': 'unknown', 'dist': 0.6, 'color': (0,0,255)}
+                
+                for name, saved_desc in descs.items():
+                    dist = np.linalg.norm([face_descriptor] - saved_desc, axis=1)
+                    if dist < last_found['dist']:
+                        last_found = {'name': name, 'dist': dist, 'color': (255,255,255)}
+                
+                cv2.rectangle(frameBGR, pt1=(face.left(), face.top()), pt2=(face.right(), face.bottom()), color=last_found['color'], thickness=2)
+                cv2.putText(frameBGR, last_found['name'] + ',' + emotion , org=(face.left(), face.top()), fontFace=cv2.FONT_HERSHEY_SIMPLEX, fontScale=1, color=last_found['color'], thickness=2)
+                # cv2.putText(frameBGR, last_found['name'] + ',' , org=(face.left(), face.top()), fontFace=cv2.FONT_HERSHEY_SIMPLEX, fontScale=1, color=last_found['color'], thickness=2)
+            
+                con_sec, con_min, con_hour = convertMillis(cap.get(cv2.CAP_PROP_POS_MSEC))
+                time_dict[emotion].append("{0}:{1}:{2}".format(con_hour, con_min, round(con_sec, 3)))
+                print("{0}:{1}:{2} {3}".format(con_hour, con_min, round(con_sec, 3), emotion))
+                # print("{0}:{1}:{2} {3}".format(con_hour, con_min, con_sec))
+
+        cv2.imshow('frame', frameBGR)
+
+    key = cv2.waitKey(25)
+    if key == 27 :
+        break
+
+print(time_dict)       
+print("총 시간 : ", time.time() - start)
+if cap.isOpened():
+    cap.release()
+
+for i in range(1,5):
+    cv2.destroyAllWindows()
+    cv2.waitKey(1)

+import dlib
+import cv2
+import numpy as np
+import matplotlib.pyplot as plt
+import tensorflow as tf
+import math
+import os
+import pathlib
+import time
+import pandas as pd
+import tensorflow as tf
+from tensorflow.keras.preprocessing.image import ImageDataGenerator,load_img
+from tensorflow.keras.models import load_model
+from tensorflow.keras import regularizers
+from tensorflow import keras
+from imutils import face_utils
+import time
+
+ 
+start = time.time()
+detector = dlib.get_frontal_face_detector()
+predictor = dlib.shape_predictor("./models/shape_predictor_68_face_landmarks.dat")
+facerec = dlib.face_recognition_model_v1('models/dlib_face_recognition_resnet_model_v1.dat')
+# model = load_model('../checkpoint/er-best-mobilenet1-bt32-model-adam.h5')
+model = load_model('../checkpoint/er-best-efficientNet1-bt32-model-SGD.h5')
+    
+    
+descs = np.load('img/descs2.npy', allow_pickle=True)[()]
+ 
+video_path = './data/zoom_1.mp4'
+cap=cv2.VideoCapture(video_path)
+ 
+
+
+# labels_dict_ = {0 : 'angry', 1 : 'fear' ,  2: 'happy', 3: 'neutral', 4:  'sad', 5: 'surprise'}
+labels_dict_ = {'angry' : 0,'fear' : 1 ,'happy' : 2, 'neutral' : 3,  'sad' : 4, 'surprise' : 5}
+time_dict = {'angry': [], 'fear': [], 'happy': [], 'neutral': [], 'sad': [], 'surprise': []}
+def get_key(val):
+    for key, value in labels_dict_.items():
+        if(value == val):
+            return key
+
+
+def convertMillis(millis):
+    seconds=(millis/1000)%60 
+    minutes=(millis/(1000*60))%60
+    hours=(millis/(1000*60*60))%24
+    return seconds, int(minutes), int(hours)
+
+#cap = cv2.VideoCapture(0) # 0번 카메라
+ 
+# 동영상 크기(frame정보)를 읽어옴
+frameWidth = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
+frameHeight = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
+frame_size = (frameWidth, frameHeight)
+fps = cap.get((cv2.CAP_PROP_FPS))
+
+
+_, img_bgr = cap.read() # (800, 1920, 3)
+padding_size = 0
+resized_width = 1920
+video_size = (resized_width, int(img_bgr.shape[0] * resized_width // img_bgr.shape[1]))
+timestamps = [cap.get(cv2.CAP_PROP_POS_MSEC)]
+prev_time = 0
+
+fourcc = cv2.VideoWriter_fourcc('m', 'p', '4', 'v')
+# out1 = cv2.VideoWriter('./data/record0.mp4',fourcc, fps, frame_size)
+
+while True:
+    retval, frameBGR = cap.read()	# 영상을 한 frame씩 읽어오기
+    current_time = time.time() - prev_time
+
+    if(type(frameBGR) == type(None)):
+        pass
+    else:
+        frameBGR = cv2.resize(frameBGR, video_size)
+        frame = cv2.cvtColor(frameBGR, cv2.COLOR_BGR2RGB)
+        
+        if (retval is True) and (current_time > 1.5) :
+            prev_time = time.time()
+            faces = detector(frame, 1)
+            
+            for (i, face) in enumerate(faces):
+                shape = predictor(frame, face)
+                face_descriptor = facerec.compute_face_descriptor(frame, shape)
+                
+                img = cv2.resize(frame[face.top():face.bottom(), face.left():face.right()], dsize=(224, 224), interpolation = cv2.INTER_CUBIC)
+                imgarr = np.array(img).reshape(1, 224, 224, 3) /255
+                # emotion = labels_dict_[model.predict(imgarr).argmax(axis=-1)[0]]
+                emotion = get_key(model.predict_classes(imgarr))
+
+                
+                last_found = {'name': 'unknown', 'dist': 0.6, 'color': (0,0,255)}
+                
+                for name, saved_desc in descs.items():
+                    dist = np.linalg.norm([face_descriptor] - saved_desc, axis=1)
+                    if dist < last_found['dist']:
+                        last_found = {'name': name, 'dist': dist, 'color': (255,255,255)}
+                
+                cv2.rectangle(frameBGR, pt1=(face.left(), face.top()), pt2=(face.right(), face.bottom()), color=last_found['color'], thickness=2)
+                cv2.putText(frameBGR, last_found['name'] + ',' + emotion , org=(face.left(), face.top()), fontFace=cv2.FONT_HERSHEY_SIMPLEX, fontScale=1, color=last_found['color'], thickness=2)
+                # cv2.putText(frameBGR, last_found['name'] + ',' , org=(face.left(), face.top()), fontFace=cv2.FONT_HERSHEY_SIMPLEX, fontScale=1, color=last_found['color'], thickness=2)
+            
+                con_sec, con_min, con_hour = convertMillis(cap.get(cv2.CAP_PROP_POS_MSEC))
+                time_dict[emotion].append("{0}:{1}:{2}".format(con_hour, con_min, round(con_sec, 3)))
+                print("{0}:{1}:{2} {3}".format(con_hour, con_min, round(con_sec, 3), emotion))
+                # print("{0}:{1}:{2} {3}".format(con_hour, con_min, con_sec))
+
+        cv2.imshow('frame', frameBGR)
+
+    key = cv2.waitKey(25)
+    if key == 27 :
+        break
+
+print(time_dict)       
+print("총 시간 : ", time.time() - start)
+if cap.isOpened():
+    cap.release()
+
+for i in range(1,5):
+    cv2.destroyAllWindows()
+    cv2.waitKey(1)