Skip to content

2
2022-074
Commit 480b705c authored by IT19165226-Arachchige I.D's avatar IT19165226-Arachchige I.D
Browse files
Options

Facial Mesh model used to identify facial landmarks and get rotational angles

parent dd809ce3
Hide whitespace changes
Inline Side-by-side
Showing with 357 additions and 0 deletions
Face Mesh - Video.ipynb 0 → 100644
View file @ 480b705c
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"id": "ddaddef7",
"metadata": {},
"outputs": [],
"source": [
"import cv2\n",
"import mediapipe as mp\n",
"import numpy as np\n",
"import time\n",
"import os\n",
"import csv\n",
"import moviepy.editor as editor\n",
"from varname import nameof\n",
"\n",
"mp_face_mesh = mp.solutions.face_mesh\n",
"face_mesh = mp_face_mesh.FaceMesh(min_detection_confidence=0.5, min_tracking_confidence=0.5)\n",
"\n",
"mp_drawing = mp.solutions.drawing_utils\n",
"\n",
"drawing_spec = mp_drawing.DrawingSpec(thickness=1, circle_radius=1)"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "6ab278cb",
"metadata": {},
"outputs": [],
"source": [
"##SEPARATING AUDIO FROM VIDEO\n",
"\n",
"def extract_audio():\n",
" #defining the clip\n",
" #r = to read the\n",
" my_clip = editor.VideoFileClip(r\"test_videos/LRH_video_01.mp4\")\n",
"\n",
" #converting\n",
" my_clip.audio.write_audiofile(r'result_analysis/LRH_video_01/my_result.mp3')\n",
"\n",
" #converting to wav - easier with speech recognition\n",
" my_clip.audio.write_audiofile(r\"result_analysis/LRH_video_01/my_result_sr.wav\")"
]
},
{
"cell_type": "code",
"execution_count": 6,
"id": "6e6b014e",
"metadata": {},
"outputs": [],
"source": [
"def get_rotational_angles():\n",
" \n",
" cap = cv2.VideoCapture(VIDEO_STREAM)\n",
"\n",
" while True:\n",
" success, image = cap.read()\n",
" \n",
" if success:\n",
"\n",
" #start = time.time()\n",
"\n",
" #flip the image horizontally to ensire it is not mirrored\n",
" #convert the color space from BGR to RGB\n",
" image = cv2.cvtColor(cv2.flip(image, 1), cv2.COLOR_BGR2RGB)\n",
"\n",
" #to improve performance\n",
" image.flags.writeable = False\n",
"\n",
" results = face_mesh.process(image)\n",
"\n",
" image.flags.writeable = True\n",
"\n",
" image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)\n",
"\n",
" img_h, img_w, img_c = image.shape #c = channels\n",
" face_3d = []\n",
" face_2d = []\n",
"\n",
" if results.multi_face_landmarks: #check if results exist\n",
" for face_landmarks in results.multi_face_landmarks: #run through all facial landmarks\n",
" for idx, ln in enumerate(face_landmarks.landmark): #get index and lm\n",
" if idx == 33 or idx == 263 or idx == 1 or idx == 61 or idx == 291 or idx == 199:\n",
" if idx == 1:\n",
" nose_2d = (ln.x * img_w, ln.y * img_h)\n",
" nose_3d = (ln.x * img_w, ln.y * img_h, ln.z * 3000)\n",
"\n",
" x, y = int(ln.x * img_w), int(ln.y * img_h)\n",
"\n",
" face_2d.append([x, y])\n",
" face_3d.append([x, y, ln.z])\n",
"\n",
" face_2d = np.array(face_2d, dtype=np.float64)\n",
"\n",
" face_3d = np.array(face_3d, dtype=np.float64) \n",
"\n",
" focal_length = 1 * img_w\n",
"\n",
" cam_matrix = np.array([ [focal_length, 0, img_h / 2],\n",
" [0, focal_length, img_w / 2],\n",
" [0, 0, 1]])\n",
"\n",
" # The distortion parameters\n",
" dist_matrix = np.zeros((4, 1), dtype=np.float64)\n",
"\n",
" # Solve PnP\n",
" success, rot_vec, trans_vec = cv2.solvePnP(face_3d, face_2d, cam_matrix, dist_matrix)\n",
"\n",
" # Get rotational matrix\n",
" rmat, jac = cv2.Rodrigues(rot_vec)\n",
"\n",
" #get angles\n",
" angles, mtxR, mtxQ, Qx, Qy, Qz = cv2.RQDecomp3x3(rmat)\n",
"\n",
" #get the rotation degree\n",
" x = angles[0] * 360\n",
" y = angles[1] * 360\n",
" z = angles[2] * 360\n",
" \n",
" record = [y, x, z] \n",
" rot_angle_records.append(record)\n",
" \n",
" mp_drawing.draw_landmarks(\n",
" image=image,\n",
" landmark_list=face_landmarks,\n",
" #connections=mp_face_mesh.FACE_CONNECTIONS,\n",
" landmark_drawing_spec=drawing_spec,\n",
" connection_drawing_spec=drawing_spec)\n",
"\n",
" cv2.imshow('Head Pose Estimation', image)\n",
" else:\n",
" print('End of video input')\n",
" break"
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "0204162e",
"metadata": {},
"outputs": [],
"source": [
"##writing details to the CSV\n",
"\n",
"def write_to_csv(details ,storage_location):\n",
" \n",
" #creating the csv template\n",
" with open(storage_location, mode='w', newline='') as headpose_track_file:\n",
" writer = csv.writer(headpose_track_file) #, delimiter=',', quotechar='\"', quoting = csv.QUOTE_MINIMAL)\n",
" \n",
" if (details == \"rot_ang\"):\n",
" writer.writerow(['Frame_number','Yaw','Pitch', 'Roll'])\n",
" \n",
" frameNo = 0\n",
"\n",
" for record in rot_angle_records:\n",
" frameNo += 1 #increase the frame number\n",
" #writing the record details to the file\n",
" writer.writerow([frameNo, record[0], record[1], record[2]])\n",
" \n",
" elif (details == \"head_turn\"):\n",
" \n",
" for name_called_frame in name_call_records:\n",
" identify_head_turn(name_called_frame)\n",
" \n",
" writer.writerow(['Name_called_frame', name_called_frame])\n",
" writer.writerow(['Name_called_time (s)', name_called_frame / 30])\n",
"\n",
" #writer.writerow(['Frame_number', 'Horizontal Change', 'Horizontal', 'Horizontal Turn', 'Horizontal Turn Time', 'Vertical Change', 'Vertical', 'Vertical Turn', 'Vertical Turn Time'])\n",
" writer.writerow(['Frame_number', 'Horizontal Change', 'Horizontal Turn', 'Horizontal Turn Time', 'Vertical Change', 'Vertical Turn', 'Vertical Turn Time'])\n",
" \n",
" frameNo = name_called_frame\n",
"\n",
" for record in head_turn_records:\n",
" frameNo += 1 #increase the frame number\n",
" #writing the record details to the file\n",
" #writer.writerow([frameNo, record[0], record[1], record[2], record[3], record[4], record[5], record[6], record[7]])\n",
" writer.writerow([frameNo, record[0], record[2], record[3], record[4], record[6], record[7]])\n",
" \n",
" writer.writerow([])"
]
},
{
"cell_type": "code",
"execution_count": 5,
"id": "5a35f03b",
"metadata": {},
"outputs": [],
"source": [
"def identify_head_turn(name_called_frame):\n",
" \n",
" hoz_count, ver_count = 0, 0\n",
" #hoz, ver = \"No change\", \"No change\"\n",
" hoz_turn, ver_turn = \"NO\", \"NO\"\n",
" hoz_turn_time, ver_turn_time = -1, -1\n",
" yaw_initial, pitch_initial = (rot_angle_records[name_called_frame])[0], (rot_angle_records[name_called_frame])[1]\n",
" last_frame = (name_called_frame + 5 * 30)\n",
" \n",
" \n",
" if len(rot_angle_records) < last_frame :\n",
" last_frame = len(rot_angle_records) - 1\n",
" \n",
" #specifying the time range to check for head turn.\n",
" ##considered only 5 seconds currently\n",
" for i in range (name_called_frame + 1, last_frame):\n",
" \n",
" hoz_diff, ver_diff = abs(abs((rot_angle_records[i])[0]) - abs(yaw_initial)), abs(abs(rot_angle_records[i][1]) - abs(pitch_initial))\n",
" \n",
" #checking for horizontal movement\n",
" if hoz_diff <- 30:\n",
" hoz = \"changed\"\n",
" #allowing system errors by checking if increase exists for at least five initial frames\n",
" hoz_count += 1\n",
" if (hoz_count == 5):\n",
" hoz_turn = \"YES\"\n",
" hoz_turn_time = i / 30\n",
" else:\n",
" hoz = \"No change\"\n",
" \n",
" #checking for vertical movement\n",
" if ver_diff <- 30:\n",
" ver_count += 1\n",
" #allowing system errors by checking if increase exists for at least five initial frame\n",
" ver = \"changed\"\n",
" if (ver_count == 5):\n",
" ver_turn = \"YES\"\n",
" ver_turn_time = i / 30\n",
" else:\n",
" ver = \"No change\"\n",
" \n",
" record = [hoz_diff, hoz, hoz_turn, hoz_turn_time, ver_diff, ver, ver_turn, ver_turn_time]\n",
" head_turn_records.append(record)"
]
},
{
"cell_type": "code",
"execution_count": 7,
"id": "a044a5c5",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"End of video input\n",
"[INFO] REMOVING AUDIO\n",
"MoviePy - Writing audio in result_analysis/LRH_video_01/my_result.mp3\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
" \r"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"MoviePy - Done.\n",
"MoviePy - Writing audio in result_analysis/LRH_video_01/my_result_sr.wav\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
" \r"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"MoviePy - Done.\n",
"[INFO] FINISHED PROCESSING\n"
]
}
],
"source": [
"#video details\n",
"VIDEO_STREAM = 'test_videos/GP01_Thineeshya_2.mp4'\n",
"video_name = 'GP01_Thineeshya_2'\n",
"output_folder = 'result_analysis/' + video_name\n",
"\n",
"#to save details of rotational angles\n",
"rot_angle_records = []\n",
"#to save details of times of name call\n",
" ## will contain frame numbers of name called\n",
"name_call_records = []\n",
"#to save details of head turns\n",
"head_turn_records = []\n",
"\n",
"#processing\n",
"get_rotational_angles()\n",
"\n",
"#saving generated data\n",
" #check if specified path exists, if not create it\n",
"if not (os.path.isdir(output_folder)):\n",
" os.makedirs(output_folder, mode = 0o777, exist_ok = False)\n",
" \n",
"print('[INFO] REMOVING AUDIO')\n",
"extract_audio()\n",
"\n",
"storage_location = output_folder + '/' + video_name + '_rotational_angles.csv'\n",
" #write rotational angles data to csv\n",
"write_to_csv(\"rot_ang\", storage_location) \n",
"\n",
"\n",
"#test data for name calls\n",
"#name_call_records = [5 * 30, 10 * 30, 13 * 30] #LRH_01\n",
"#name_call_records = [8 * 30] #Kian\n",
"name_call_records = [1 * 30] #GP01_Thineeshya_1 / 2\n",
"\n",
"storage_location = output_folder + '/' + video_name + '_head_turns.csv'\n",
" #write head turns identified data to csv\n",
" ##will identify head turns inside\n",
"write_to_csv(\"head_turn\", storage_location) \n",
"\n",
"print('[INFO] FINISHED PROCESSING')"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "706be793",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.9.7"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment